Accessing and Querying the Survey Class

The survey HDF5 file is a master look up table for each observation made by VIRUS for HETDEX. It can be accessed directly through the HDF5 survey file, or through the Survey class API. The Survey class includes information for each three dither set HETDEX observation in HDR1. It includes both science and calibration shots. Any quantity related to an individual dither is provided in an array of length three. Fundamental properties of a shot and quality assessment values for seeing (fwhm), astrometry, flux limits and transparency (response_4540) are tabulated to allow for querying of desired shots.

We first introduce the basic API to query the Survey Class by sky coordinates to get a list of shots where fibers are located within that region. Later on in this notebook we show some basic querying of the Survey class and demonstrate a few examples.

Initiate the Survey class

%matplotlib inline
import tables as tb
import numpy as np
import matplotlib.pyplot as plt

from astropy.table import Table
import astropy.units as u
from astropy.coordinates import SkyCoord

from hetdex_api.survey import Survey, FiberIndex

S = Survey('pdr1')

# convert to an astropy table format. Can remove any flagged shots if desired by setting return_good=True.
survey_table = S.return_astropy_table(return_good=False) 

survey_table

Table length=3502

shotid	field	objid	date	obsid	ra	dec	pa	n_ifu	fwhm_virus	fwhm_virus_err	response_4540	ambtemp	datevobs	dewpoint	exptime	humidity	mjd	nstars_fit_fwhm	obsind	pressure	structaz	time	trajcdec	trajcpa	trajcra	shot_flag
int64	bytes12	bytes18	int32	int32	float64	float64	float64	int32	float32	float32	float32	float32	bytes12	float32	float32	float32	float32	int32	int32	float32	float32	bytes7	float32	float32	float32	bool
20170920008	dex-fall	DEX00068_0_000_E	20170920	8	24.588702	0.000469	154.144974	21	1.6656576	0.40102777	0.1212	20.422777	20170920v008	10.840555	367.09952	52.572	58016.305	8	1	802.2015	147.76714	0717562	0.0	152.69672	24.5895	True
20170920009	dex-fall	DEX00094_0_000_E	20170920	9	31.608569	0.000689	154.184946	21	1.9848523	0.18202426	0.1006	20.372223	20170920v009	10.787222	367.59937	52.337	58016.324	4	1	802.2015	147.78941	0745268	0.0	152.71495	31.6095	True
20170921018	dex-fall	DEX00001_0	20170921	18	6.498122	0.000526	154.177847	18	1.7578322	0.5647613	0.1374	20.896667	20170921v018	9.74	367.166	46.673	58017.25	7	1	803.082	147.73189	0559186	0.0	152.66785	6.4995	True
20170921020	dex-fall	DEX00046_0_000_E	20170921	20	18.649292	0.001456	154.126565	18	1.6957773	0.27283788	0.113	20.472221	20170921v020	10.080556	367.06583	49.737	58017.285	12	1	803.082	147.75246	0647548	0.0	152.6847	18.6495	True
20170921021	dex-fall	DEX00070_0_000_E	20170921	21	25.127779	0.000856	154.16784	18	1.6449038	0.18297766	0.118	20.043333	20170921v021	10.283889	366.5493	51.741	58017.3	4	1	803.0481	147.76953	0712281	0.0	152.69867	25.1295	True
20170921022	dex-fall	DEX00070_0_000_E	20170921	22	31.338945	0.000353	154.178197	18	1.9767525	0.5830754	0.1277	19.638334	20170921v022	10.735556	367.19894	54.857	58017.316	7	1	803.0481	147.78941	0736586	0.0	152.71495	31.3395	True
20170921025	dex-fall	DEX00017_1_000_W	20170921	25	10.820815	0.000884	208.837196	18	1.7373962	0.14883323	0.0892	19.426111	20170921v025	10.836667	366.96646	56.025	58017.36	4	1	802.91266	212.2687	0838279	0.0	207.33266	10.8195	True
20170921026	dex-fall	DEX00041_1_000_W	20170921	26	17.300609	0.001184	208.919681	18	1.7994181	0.013720359	0.116	18.890556	20170921v026	10.827222	367.54956	58.009	58017.38	5	1	802.7095	212.24982	0905028	0.0	207.31718	17.2995	True
20170921029	dex-fall	DEX00095_1_000_W	20170921	29	31.878935	0.000924	208.749334	18	1.8989948	0.898437	0.1213	18.762777	20170921v029	10.489445	367.14957	57.425	58017.42	6	1	802.1676	212.20863	1006376	0.0	207.28346	31.8795	True
20170922021	dex-fall	DEX00015_0_000_E	20170922	21	10.278399	0.001199	154.164499	21	1.8325357	0.5471163	0.1356	20.06611	20170922v021	11.661667	366.7827	57.33	58018.26	10	1	801.38873	147.73499	0616563	0.0	152.67038	10.2795	True
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
20210810022	dex-fall	DEXfl1201	20210810	22	15.747345	1.386349	150.335036	77	1.6856463	0.24644043	0.137	20.44111	20210810v022	12.118889	368.47778	56.008	59436.383	22	1	806.0958	143.12718	0909389	1.386358	148.92297	15.74763	True
20210810023	dex-fall	DEXfl0547	20210810	23	20.751032	1.387937	150.333082	77	1.4781301	0.21766174	0.155	20.301666	20210810v023	12.070556	427.60727	55.989	59436.4	31	1	806.0958	143.12718	0932480	1.387939	148.92297	20.75196	True
20210810024	dex-fall	DEXfl1831	20210810	24	27.750462	1.383569	150.305894	77	1.5440501	0.20046702	0.141	20.018888	20210810v024	11.161111	367.36584	53.706	59436.418	24	1	806.23126	143.12718	0959245	1.383825	148.92297	27.750586	True
20210818008	dex-fall	DEXfl0695	20210818	8	24.49757	2.359227	147.751397	77	1.7026963	0.47975463	0.133	16.936666	20210818v008	13.848889	368.33145	81.192	59444.383	11	1	803.556	139.96121	0908316	2.359597	146.37807	24.49788	True
20210818009	dex-fall	DEXfl1966	20210818	9	30.50176	2.358941	147.778205	77	1.6296377	0.19967884	0.157	17.472778	20210818v009	13.051666	428.275	73.847	59444.4	20	1	803.556	139.96121	0932311	2.359597	146.37807	30.502125	True
20210818010	dex-fall	DEXfl1901	20210818	10	34.997393	1.803343	147.794806	77	1.3866715	0.20080687	0.122	17.562778	20210818v010	12.544444	368.19986	71.304	59444.418	28	1	803.6915	139.96121	0959180	1.803569	146.37807	34.997623	True
20210818011	dex-fall	DEXfl0791	20210818	11	8.751763	-1.662152	202.591086	77	1.4317216	0.23261194	0.14	18.206112	20210818v011	11.935555	428.2084	66.21	59444.434	21	1	803.6915	204.77533	1023392	-1.662814	201.13431	8.75208	True
20210826008	dex-spring	DEXsp6607	20210826	8	240.071344	52.579344	295.52774	78	1.4278307	0.15777344	0.158	20.915556	20210826v008	12.225	367.3104	54.519	59452.12	37	1	807.3488	319.74277	0251417	52.57897	293.8813	240.07066	True
20210828007	dex-spring	DEXsp6643	20210828	7	240.801501	52.778936	295.966469	78	1.3569877	0.14228329	0.146	21.441668	20210828v007	11.168889	367.7404	48.037	59454.117	39	1	804.5042	320.1262	0247291	52.778416	294.31314	240.80075	True
20210830013	dex-spring	DEXsp6649	20210830	13	244.692454	50.99492	293.60186	78	1.5250597	0.28602806	0.145	19.721666	20210830v013	14.098333	487.7771	67.078	59456.137	31	1	805.24927	317.97006	0316036	50.9942	291.92062	244.69333	True

It also converts RA and DEC into astropy coordinate objects.

S.coords

<SkyCoord (ICRS): (ra, dec) in deg
    [( 24.588702, 4.6900000e-04), ( 31.608569, 6.8900000e-04),
     (  6.498122, 5.2600000e-04), ..., (240.071344, 5.2579344e+01),
     (240.801501, 5.2778936e+01), (244.692454, 5.0994920e+01)]>

Shots are labeled based on their observation date and observation ID:

S.date

array([20170920, 20170920, 20170921, ..., 20210826, 20210828, 20210830],
      dtype=int32)

S.obsid

array([ 8,  9, 18, ...,  8,  7, 13], dtype=int32)

A specific shot can be linked either by an integer combination of these two values, called shotid, or the DATEvOBS form:

S.datevobs

array(['20170920v008', '20170920v009', '20170921v018', ...,
       '20210826v008', '20210828v007', '20210830v013'], dtype='<U12')

S.shotid

array([20170920008, 20170920009, 20170921018, ..., 20210826008,
       20210828007, 20210830013])

Searching for shots near a sky coordinate wiht get_shotlist

Most users searching for observations will want to get a list of observations/shots at a specific sky position. Each shot has a single associated sky coordinate. We can retrieve a list of shots that may contain fibers in a specific region. However, because of the varying and non-contiguous footprint of VIRUS, this is not an immediate guarantee a region has been observed. You must further query fibers in each shot to determine this. See Notebook 02 to learn how to query with the Fibers Class API.

This function which operates on the Survey class object. An astropy coordinates object is required as input.

coords = SkyCoord(11.628530 * u.deg, 0.081790 * u.deg, frame='icrs')

We can either search a rectangular region, width and height in degrees:

shotlist = S.get_shotlist(coords, width=0.5, height=0.2)

shotlist

array([20171119003, 20180106004, 20181003009, 20181005017, 20181107014,
       20191222012])

or we can search a circular aperture with a radius given in degress, or in an astropy quantity object:

shotlist = S.get_shotlist(coords, radius=10*u.arcmin)

shotlist

array([20171119003, 20180106004, 20181003009, 20181107014, 20191222012])

From here a user can take their shotlist and query the position on the Fibers class and see if their region of interest is observed in the survey.

Basic querying for the Survey Class

The Survey HDF5 itself contains information about the quality of each observation. A user can query different parameters and plot up general shot quantities. For example, we can compare the distribution of throughput values (technically the response at 4540 AA) between the HETDEX spring and HETDEX fall fields:

Example: Plotting up the canonical throughput value at 4540 AA

idx_spring = np.where( (S.field == 'dex-spring') * np.isfinite(S.response_4540) )

idx_fall = np.where( (S.field == 'dex-fall') * np.isfinite(S.response_4540))

plt.figure(figsize=(8,6))
plt.hist(S.response_4540[idx_spring], label = 'DEX-Spring', bins=20)
plt.hist(S.response_4540[idx_fall], label = 'DEX-Fall', bins=20)
plt.xlabel('Throughput at 4540 AA')
plt.ylabel('N OBS')
plt.legend()

<matplotlib.legend.Legend at 0x15045f2d7d90>

Access the survey H5 file

The hdfile attribute contains the link to the survey HDF5 file. Calling it will show you all the information in the HDF5 file. You may interact with this directly rather than using the Survey class initiation.

S.hdfile

File(filename=/scratch/projects/hetdex/pdr1/survey/survey_pdr1.h5, title='PDR1 Survey File', mode='r', root_uep='/', filters=Filters(complevel=0, shuffle=False, bitshuffle=False, fletcher32=False, least_significant_digit=None))
/ (RootGroup) 'PDR1 Survey File'
/Survey (Table(3502,)) ''
  description := {
  "shotid": Int64Col(shape=(), dflt=0, pos=0),
  "field": StringCol(itemsize=12, shape=(), dflt=b'', pos=1),
  "objid": StringCol(itemsize=18, shape=(), dflt=b'', pos=2),
  "date": Int32Col(shape=(), dflt=0, pos=3),
  "obsid": Int32Col(shape=(), dflt=0, pos=4),
  "ra": Float64Col(shape=(), dflt=0.0, pos=5),
  "dec": Float64Col(shape=(), dflt=0.0, pos=6),
  "pa": Float64Col(shape=(), dflt=0.0, pos=7),
  "n_ifu": Int32Col(shape=(), dflt=0, pos=8),
  "fwhm_virus": Float32Col(shape=(), dflt=0.0, pos=9),
  "fwhm_virus_err": Float32Col(shape=(), dflt=0.0, pos=10),
  "response_4540": Float32Col(shape=(), dflt=0.0, pos=11),
  "ambtemp": Float32Col(shape=(), dflt=0.0, pos=12),
  "datevobs": StringCol(itemsize=12, shape=(), dflt=b'', pos=13),
  "dewpoint": Float32Col(shape=(), dflt=0.0, pos=14),
  "exptime": Float32Col(shape=(), dflt=0.0, pos=15),
  "humidity": Float32Col(shape=(), dflt=0.0, pos=16),
  "mjd": Float32Col(shape=(), dflt=0.0, pos=17),
  "nstars_fit_fwhm": Int32Col(shape=(), dflt=0, pos=18),
  "obsind": Int32Col(shape=(), dflt=0, pos=19),
  "pressure": Float32Col(shape=(), dflt=0.0, pos=20),
  "structaz": Float32Col(shape=(), dflt=0.0, pos=21),
  "time": StringCol(itemsize=7, shape=(), dflt=b'', pos=22),
  "trajcdec": Float32Col(shape=(), dflt=0.0, pos=23),
  "trajcpa": Float32Col(shape=(), dflt=0.0, pos=24),
  "trajcra": Float32Col(shape=(), dflt=0.0, pos=25),
  "shot_flag": BoolCol(shape=(), dflt=False, pos=26)}
  byteorder := 'little'
  chunkshape := (425,)

Close the Survey H5 file after use

It is good practice to close any h5 files that are open when you no longer need to access new data from the h5 file

S.close()