diff --git a/bin/bricks_load.py b/bin/bricks_load.py index fdec4b0bf..aaf56a179 100644 --- a/bin/bricks_load.py +++ b/bin/bricks_load.py @@ -1,9 +1,9 @@ #----------------------------------------------------------------------# # filename: pyfits+psycopg2_ex.py # author: Peter Nugent -# date: 10/30/2014 +# date: 1/29/2016 # ---------------------------------------------------------------------# -# Function: Read in a Arjun fits binary table from standard in and load it +# Function: DR2, Read in a Arjun fits binary table from standard in and load it # into the desi calib pg table database with psycopg2. # ---------------------------------------------------------------------# @@ -43,29 +43,14 @@ # Fire up the db -con = psycopg2.connect(host='scidb2.nersc.gov', user='desi_admin', password='L00cy-1959', database='desi') +con = psycopg2.connect(host='scidb2.nersc.gov', user='desi_admin', database='desi') cursor = con.cursor() # Re-cast as strings so the load is easy for i in range(0, nrows): - if tbdata['has_g'][i] == 70: - g = 0 - else: - g = 1 - - if tbdata['has_r'][i] == 70: - r = 0 - else: - r = 1 - - if tbdata['has_z'][i] == 70: - z = 0 - else: - z = 1 - - line = [ tbdata['brickname'][i], tbdata['brickid'][i], tbdata['brickrow'][i], tbdata['brickcol'][i], tbdata['brickq'][i], tbdata['ra'][i], tbdata['dec'][i], tbdata['ra1'][i], tbdata['dec1'][i], tbdata['ra2'][i], tbdata['dec2'][i], bool(g), bool(r), bool(z) ] + line = [ tbdata['brickname'][i], tbdata['brickid'][i], tbdata['brickrow'][i], tbdata['brickcol'][i], tbdata['brickq'][i], tbdata['ra'][i], tbdata['dec'][i], tbdata['ra1'][i], tbdata['dec1'][i], tbdata['ra2'][i], tbdata['dec2'][i], tbdata['nobs_med_g'][i], tbdata['nobs_med_r'][i], tbdata['nobs_med_z'][i], tbdata['nobs_max_g'][i], tbdata['nobs_max_r'][i], tbdata['nobs_max_z'][i] ] newdata.append(line) @@ -73,13 +58,13 @@ # for i, f in enumerate(newdata): ## - query = 'INSERT INTO bricks ( brickname, brickid, brickrow, brickcol, brickq, ra, dec, ra1, dec1, ra2, dec2, has_g, has_r, has_z) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )' + query = 'INSERT INTO bricks ( brickname, brickid, brickrow, brickcol, brickq, ra, dec, ra1, dec1, ra2, dec2, nobs_med_g, nobs_med_r, nobs_med_z, nobs_max_g, nobs_max_r, nobs_max_z) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )' ## cursor.execute( query, tuple( [str(elem) for elem in newdata[i]] ) ) # # con.commit() - +print 'done' # That's it! diff --git a/bin/tractor_load.py b/bin/tractor_load.py index 4a3c9e2eb..d98ab58f6 100644 --- a/bin/tractor_load.py +++ b/bin/tractor_load.py @@ -1,9 +1,9 @@ #----------------------------------------------------------------------# # filename: astropy+psycopg2_ex.py # author: Peter Nugent -# date: 2/18/2015 +# date: 1/29/2016 # ---------------------------------------------------------------------# -# Function: Read in a Dustin's fits tractor binary table from standard +# Function: DR2, Read in a Dustin's fits tractor binary table from standard # in and load it into the desi candidate pg table database with psycopg2. # ---------------------------------------------------------------------# @@ -33,13 +33,13 @@ # # Fire up the db # -con = psycopg2.connect(host='scidb2.nersc.gov', user='desi_admin', password='L00cy-1959', database='desi') +con = psycopg2.connect(host='scidb2.nersc.gov', user='desi_admin', database='desi') cursor = con.cursor() prihdr=table[0].header -tractver = prihdr['TRACTORV'] +tractver = prihdr['LEGPIPEV'] hdrbrickid = prihdr['brickid'] ## @@ -47,7 +47,7 @@ loaded = cursor.fetchone()[0] if loaded: - print fimage, 'is already loaded' + print 'SKIPPING ',fimage,'ALREADY loaded it' sys.exit(0) ## @@ -68,7 +68,6 @@ newdata_decap = [] - for i in range(0, nrows): if tbdata['brick_primary'][i] == 84: @@ -82,15 +81,22 @@ oob = 0 else: oob = 1 + + if tbdata['tycho2inblob'][i] == 70: + tib = 0 + else: + tib = 1 - line_cand = [ tbdata['brickid'][i], tbdata['objid'][i], tbdata['blob'][i], tbdata['type'][i], tbdata['ra'][i], tbdata['ra_ivar'][i], tbdata['dec'][i], tbdata['dec_ivar'][i], tbdata['bx'][i], tbdata['by'][i], tbdata['bx0'][i], tbdata['by0'][i], bool(lb), bool(oob), tbdata['ebv'][i], tbdata['dchisq'][i][0], tbdata['dchisq'][i][1], tbdata['dchisq'][i][2], tbdata['dchisq'][i][3], tbdata['fracDev'][i], tbdata['fracDev_ivar'][i], tbdata['shapeExp_r'][i], tbdata['shapeExp_r_ivar'][i], tbdata['shapeExp_e1'][i], tbdata['shapeExp_e1_ivar'][i], tbdata['shapeExp_e2'][i], tbdata['shapeExp_e2_ivar'][i], tbdata['shapeDev_r'][i], tbdata['shapeDev_r_ivar'][i], tbdata['shapeDev_e1'][i], tbdata['shapeDev_e1_ivar'][i], tbdata['shapeDev_e2'][i], tbdata['shapeDev_e2_ivar'][i] ] + line_cand = [ tbdata['brickid'][i], tbdata['objid'][i], tbdata['blob'][i], tbdata['ninblob'][i], bool(tib), tbdata['type'][i], tbdata['ra'][i], tbdata['ra_ivar'][i], tbdata['dec'][i], tbdata['dec_ivar'][i], tbdata['bx'][i], tbdata['by'][i], tbdata['bx0'][i], tbdata['by0'][i], bool(lb), bool(oob), tbdata['ebv'][i], tbdata['dchisq'][i][0], tbdata['dchisq'][i][1], tbdata['dchisq'][i][2], tbdata['dchisq'][i][3], tbdata['dchisq'][i][4], tbdata['fracDev'][i], tbdata['fracDev_ivar'][i], tbdata['shapeExp_r'][i], tbdata['shapeExp_r_ivar'][i], tbdata['shapeExp_e1'][i], tbdata['shapeExp_e1_ivar'][i], tbdata['shapeExp_e2'][i], tbdata['shapeExp_e2_ivar'][i], tbdata['shapeDev_r'][i], tbdata['shapeDev_r_ivar'][i], tbdata['shapeDev_e1'][i], tbdata['shapeDev_e1_ivar'][i], tbdata['shapeDev_e2'][i], tbdata['shapeDev_e2_ivar'][i] ] - line_decam = [ tbdata['decam_flux'][i][0], tbdata['decam_flux_ivar'][i][0], tbdata['decam_fracflux'][i][0], tbdata['decam_fracmasked'][i][0], tbdata['decam_fracin'][i][0], tbdata['decam_rchi2'][i][0], tbdata['decam_nobs'][i][0], tbdata['decam_anymask'][i][0], tbdata['decam_allmask'][i][0], tbdata['decam_mw_transmission'][i][0], tbdata['decam_flux'][i][1], tbdata['decam_flux_ivar'][i][1], tbdata['decam_fracflux'][i][1], tbdata['decam_fracmasked'][i][1], tbdata['decam_fracin'][i][1], tbdata['decam_rchi2'][i][1], tbdata['decam_nobs'][i][1], tbdata['decam_anymask'][i][1], tbdata['decam_allmask'][i][1], tbdata['decam_mw_transmission'][i][1], tbdata['decam_flux'][i][2], tbdata['decam_flux_ivar'][i][2], tbdata['decam_fracflux'][i][2], tbdata['decam_fracmasked'][i][2], tbdata['decam_fracin'][i][2], tbdata['decam_rchi2'][i][2], tbdata['decam_nobs'][i][2], tbdata['decam_anymask'][i][2], tbdata['decam_allmask'][i][2], tbdata['decam_mw_transmission'][i][2], tbdata['decam_flux'][i][3], tbdata['decam_flux_ivar'][i][3], tbdata['decam_fracflux'][i][3], tbdata['decam_fracmasked'][i][3], tbdata['decam_fracin'][i][3], tbdata['decam_rchi2'][i][3], tbdata['decam_nobs'][i][3], tbdata['decam_anymask'][i][3], tbdata['decam_allmask'][i][3], tbdata['decam_mw_transmission'][i][3], tbdata['decam_flux'][i][4], tbdata['decam_flux_ivar'][i][4], tbdata['decam_fracflux'][i][4], tbdata['decam_fracmasked'][i][4], tbdata['decam_fracin'][i][4], tbdata['decam_rchi2'][i][4], tbdata['decam_nobs'][i][4], tbdata['decam_anymask'][i][4], tbdata['decam_allmask'][i][4], tbdata['decam_mw_transmission'][i][4], tbdata['decam_flux'][i][5], tbdata['decam_flux_ivar'][i][5], tbdata['decam_fracflux'][i][5], tbdata['decam_fracmasked'][i][5], tbdata['decam_fracin'][i][5], tbdata['decam_rchi2'][i][5], tbdata['decam_nobs'][i][5], tbdata['decam_anymask'][i][5], tbdata['decam_allmask'][i][5], tbdata['decam_mw_transmission'][i][5] ] + line_decam = [ tbdata['decam_flux'][i][0], tbdata['decam_flux_ivar'][i][0], tbdata['decam_fracflux'][i][0], tbdata['decam_fracmasked'][i][0], tbdata['decam_fracin'][i][0], tbdata['decam_rchi2'][i][0], tbdata['decam_nobs'][i][0], tbdata['decam_anymask'][i][0], tbdata['decam_allmask'][i][0],tbdata['decam_psfsize'][i][0], tbdata['decam_mw_transmission'][i][0], tbdata['decam_depth'][i][0], tbdata['decam_galdepth'][i][0], tbdata['decam_flux'][i][1], tbdata['decam_flux_ivar'][i][1], tbdata['decam_fracflux'][i][1], tbdata['decam_fracmasked'][i][1], tbdata['decam_fracin'][i][1], tbdata['decam_rchi2'][i][1], tbdata['decam_nobs'][i][1], tbdata['decam_anymask'][i][1], tbdata['decam_allmask'][i][1], tbdata['decam_psfsize'][i][1], tbdata['decam_mw_transmission'][i][1],tbdata['decam_depth'][i][1],tbdata['decam_galdepth'][i][1], tbdata['decam_flux'][i][2], tbdata['decam_flux_ivar'][i][2], tbdata['decam_fracflux'][i][2], tbdata['decam_fracmasked'][i][2], tbdata['decam_fracin'][i][2], tbdata['decam_rchi2'][i][2], tbdata['decam_nobs'][i][2], tbdata['decam_anymask'][i][2], tbdata['decam_allmask'][i][2], tbdata['decam_psfsize'][i][2], tbdata['decam_mw_transmission'][i][2], tbdata['decam_depth'][i][2],tbdata['decam_galdepth'][i][2], tbdata['decam_flux'][i][3], tbdata['decam_flux_ivar'][i][3], tbdata['decam_fracflux'][i][3], tbdata['decam_fracmasked'][i][3], tbdata['decam_fracin'][i][3], tbdata['decam_rchi2'][i][3], tbdata['decam_nobs'][i][3], tbdata['decam_anymask'][i][3], tbdata['decam_allmask'][i][3], tbdata['decam_psfsize'][i][3], tbdata['decam_mw_transmission'][i][3], tbdata['decam_depth'][i][3],tbdata['decam_galdepth'][i][3], tbdata['decam_flux'][i][4], tbdata['decam_flux_ivar'][i][4], tbdata['decam_fracflux'][i][4], tbdata['decam_fracmasked'][i][4], tbdata['decam_fracin'][i][4], tbdata['decam_rchi2'][i][4], tbdata['decam_nobs'][i][4], tbdata['decam_anymask'][i][4], tbdata['decam_allmask'][i][4], tbdata['decam_psfsize'][i][4], tbdata['decam_mw_transmission'][i][4], tbdata['decam_depth'][i][4],tbdata['decam_galdepth'][i][4], tbdata['decam_flux'][i][5], tbdata['decam_flux_ivar'][i][5], tbdata['decam_fracflux'][i][5], tbdata['decam_fracmasked'][i][5], tbdata['decam_fracin'][i][5], tbdata['decam_rchi2'][i][5], tbdata['decam_nobs'][i][5], tbdata['decam_anymask'][i][5], tbdata['decam_allmask'][i][5], tbdata['decam_psfsize'][i][5], tbdata['decam_mw_transmission'][i][5], tbdata['decam_depth'][i][5],tbdata['decam_galdepth'][i][5] ] line_wise = [ tbdata['wise_flux'][i][0], tbdata['wise_flux_ivar'][i][0], tbdata['wise_fracflux'][i][0], tbdata['wise_rchi2'][i][0], tbdata['wise_nobs'][i][0], tbdata['wise_mw_transmission'][i][0], tbdata['wise_flux'][i][1], tbdata['wise_flux_ivar'][i][1], tbdata['wise_fracflux'][i][1], tbdata['wise_rchi2'][i][1], tbdata['wise_nobs'][i][1], tbdata['wise_mw_transmission'][i][1], tbdata['wise_flux'][i][2], tbdata['wise_flux_ivar'][i][2], tbdata['wise_fracflux'][i][2], tbdata['wise_rchi2'][i][2], tbdata['wise_nobs'][i][2], tbdata['wise_mw_transmission'][i][2], tbdata['wise_flux'][i][3], tbdata['wise_flux_ivar'][i][3], tbdata['wise_fracflux'][i][3], tbdata['wise_rchi2'][i][3], tbdata['wise_nobs'][i][3], tbdata['wise_mw_transmission'][i][3] ] - line_decap = [ tbdata['decam_apflux'][i][0], tbdata['decam_apflux'][i][1], tbdata['decam_apflux'][i][2], tbdata['decam_apflux'][i][3], tbdata['decam_apflux'][i][4], tbdata['decam_apflux'][i][5], tbdata['decam_apflux'][i][6], tbdata['decam_apflux'][i][7], tbdata['decam_apflux_resid'][i][0], tbdata['decam_apflux_resid'][i][1], tbdata['decam_apflux_resid'][i][2], tbdata['decam_apflux_resid'][i][3], tbdata['decam_apflux_resid'][i][4], tbdata['decam_apflux_resid'][i][5], tbdata['decam_apflux_resid'][i][6], tbdata['decam_apflux_resid'][i][7], tbdata['decam_apflux_ivar'][i][0], tbdata['decam_apflux_ivar'][i][1], tbdata['decam_apflux_ivar'][i][2], tbdata['decam_apflux_ivar'][i][3], tbdata['decam_apflux_ivar'][i][4], tbdata['decam_apflux_ivar'][i][5], tbdata['decam_apflux_ivar'][i][6], tbdata['decam_apflux_ivar'][i][7], tbdata['decam_apflux'][i][8], tbdata['decam_apflux'][i][9], tbdata['decam_apflux'][i][10], tbdata['decam_apflux'][i][11], tbdata['decam_apflux'][i][12], tbdata['decam_apflux'][i][13], tbdata['decam_apflux'][i][14], tbdata['decam_apflux'][i][15], tbdata['decam_apflux_resid'][i][8], tbdata['decam_apflux_resid'][i][9], tbdata['decam_apflux_resid'][i][10], tbdata['decam_apflux_resid'][i][11], tbdata['decam_apflux_resid'][i][12], tbdata['decam_apflux_resid'][i][13], tbdata['decam_apflux_resid'][i][14], tbdata['decam_apflux_resid'][i][15], tbdata['decam_apflux_ivar'][i][8], tbdata['decam_apflux_ivar'][i][9], tbdata['decam_apflux_ivar'][i][10], tbdata['decam_apflux_ivar'][i][11], tbdata['decam_apflux_ivar'][i][12], tbdata['decam_apflux_ivar'][i][13], tbdata['decam_apflux_ivar'][i][14], tbdata['decam_apflux_ivar'][i][15], tbdata['decam_apflux'][i][16], tbdata['decam_apflux'][i][17], tbdata['decam_apflux'][i][18], tbdata['decam_apflux'][i][19], tbdata['decam_apflux'][i][20], tbdata['decam_apflux'][i][21], tbdata['decam_apflux'][i][22], tbdata['decam_apflux'][i][23], tbdata['decam_apflux_resid'][i][16], tbdata['decam_apflux_resid'][i][17], tbdata['decam_apflux_resid'][i][18], tbdata['decam_apflux_resid'][i][19], tbdata['decam_apflux_resid'][i][20], tbdata['decam_apflux_resid'][i][21], tbdata['decam_apflux_resid'][i][22], tbdata['decam_apflux_resid'][i][23], tbdata['decam_apflux_ivar'][i][16], tbdata['decam_apflux_ivar'][i][17], tbdata['decam_apflux_ivar'][i][18], tbdata['decam_apflux_ivar'][i][19], tbdata['decam_apflux_ivar'][i][20], tbdata['decam_apflux_ivar'][i][21], tbdata['decam_apflux_ivar'][i][22], tbdata['decam_apflux_ivar'][i][23], tbdata['decam_apflux'][i][24], tbdata['decam_apflux'][i][25], tbdata['decam_apflux'][i][26], tbdata['decam_apflux'][i][27], tbdata['decam_apflux'][i][28], tbdata['decam_apflux'][i][29], tbdata['decam_apflux'][i][30], tbdata['decam_apflux'][i][31], tbdata['decam_apflux_resid'][i][24], tbdata['decam_apflux_resid'][i][25], tbdata['decam_apflux_resid'][i][26], tbdata['decam_apflux_resid'][i][27], tbdata['decam_apflux_resid'][i][28], tbdata['decam_apflux_resid'][i][29], tbdata['decam_apflux_resid'][i][30], tbdata['decam_apflux_resid'][i][31], tbdata['decam_apflux_ivar'][i][24], tbdata['decam_apflux_ivar'][i][25], tbdata['decam_apflux_ivar'][i][26], tbdata['decam_apflux_ivar'][i][27], tbdata['decam_apflux_ivar'][i][28], tbdata['decam_apflux_ivar'][i][29], tbdata['decam_apflux_ivar'][i][30], tbdata['decam_apflux_ivar'][i][31], tbdata['decam_apflux'][i][32], tbdata['decam_apflux'][i][33], tbdata['decam_apflux'][i][34], tbdata['decam_apflux'][i][35], tbdata['decam_apflux'][i][36], tbdata['decam_apflux'][i][37], tbdata['decam_apflux'][i][38], tbdata['decam_apflux'][i][39], tbdata['decam_apflux_resid'][i][32], tbdata['decam_apflux_resid'][i][33], tbdata['decam_apflux_resid'][i][34], tbdata['decam_apflux_resid'][i][35], tbdata['decam_apflux_resid'][i][36], tbdata['decam_apflux_resid'][i][37], tbdata['decam_apflux_resid'][i][38], tbdata['decam_apflux_resid'][i][39], tbdata['decam_apflux_ivar'][i][32], tbdata['decam_apflux_ivar'][i][33], tbdata['decam_apflux_ivar'][i][34], tbdata['decam_apflux_ivar'][i][35], tbdata['decam_apflux_ivar'][i][36], tbdata['decam_apflux_ivar'][i][37], tbdata['decam_apflux_ivar'][i][38], tbdata['decam_apflux_ivar'][i][39], tbdata['decam_apflux'][i][40], tbdata['decam_apflux'][i][41], tbdata['decam_apflux'][i][42], tbdata['decam_apflux'][i][43], tbdata['decam_apflux'][i][44], tbdata['decam_apflux'][i][45], tbdata['decam_apflux'][i][46], tbdata['decam_apflux'][i][47], tbdata['decam_apflux_resid'][i][40], tbdata['decam_apflux_resid'][i][41], tbdata['decam_apflux_resid'][i][42], tbdata['decam_apflux_resid'][i][43], tbdata['decam_apflux_resid'][i][44], tbdata['decam_apflux_resid'][i][45], tbdata['decam_apflux_resid'][i][46], tbdata['decam_apflux_resid'][i][47], tbdata['decam_apflux_ivar'][i][40], tbdata['decam_apflux_ivar'][i][41], tbdata['decam_apflux_ivar'][i][42], tbdata['decam_apflux_ivar'][i][43], tbdata['decam_apflux_ivar'][i][44], tbdata['decam_apflux_ivar'][i][45], tbdata['decam_apflux_ivar'][i][46], tbdata['decam_apflux_ivar'][i][47] ] + line_decap=[] + for band in range(len('ugrizy')): #indices 0->5 are ugrizy in u->y order + line_decap+= [ tbdata['decam_apflux'][i][band][0], tbdata['decam_apflux'][i][band][1], tbdata['decam_apflux'][i][band][2], tbdata['decam_apflux'][i][band][3], tbdata['decam_apflux'][i][band][4], tbdata['decam_apflux'][i][band][5], tbdata['decam_apflux'][i][band][6], tbdata['decam_apflux'][i][band][7], tbdata['decam_apflux_resid'][i][band][0], tbdata['decam_apflux_resid'][i][band][1], tbdata['decam_apflux_resid'][i][band][2], tbdata['decam_apflux_resid'][i][band][3], tbdata['decam_apflux_resid'][i][band][4], tbdata['decam_apflux_resid'][i][band][5], tbdata['decam_apflux_resid'][i][band][6], tbdata['decam_apflux_resid'][i][band][7], tbdata['decam_apflux_ivar'][i][band][0], tbdata['decam_apflux_ivar'][i][band][1], tbdata['decam_apflux_ivar'][i][band][2], tbdata['decam_apflux_ivar'][i][band][3], tbdata['decam_apflux_ivar'][i][band][4], tbdata['decam_apflux_ivar'][i][band][5], tbdata['decam_apflux_ivar'][i][band][6], tbdata['decam_apflux_ivar'][i][band][7] ] newdata_cand.append(line_cand) newdata_decam.append(line_decam) @@ -102,29 +108,29 @@ ## Re-cast as strings so the load is easy # for i, f in enumerate(newdata_cand): -## - query = 'INSERT INTO candidate ( brickid, objid, blob, type, ra, ra_ivar, dec, dec_ivar, bx, by, bx0, by0, left_blob, out_of_bounds, ebv, dchisq1, dchisq2, dchisq3, dchisq4, fracdev, fracdev_ivar, shapeexp_r, shapeexp_r_ivar, shapeexp_e1, shapeexp_e1_ivar, shapeexp_e2, shapeexp_e2_ivar, shapedev_r, shapedev_r_ivar, shapedev_e1, shapedev_e1_ivar, shapedev_e2, shapedev_e2_ivar ) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) RETURNING id' -## +### + query = 'INSERT INTO candidate ( brickid, objid, blob, ninblob, tycho2inblob, type, ra, ra_ivar, dec, dec_ivar, bx, by, bx0, by0, left_blob, out_of_bounds, ebv, dchisq1, dchisq2, dchisq3, dchisq4, dchisq5, fracdev, fracdev_ivar, shapeexp_r, shapeexp_r_ivar, shapeexp_e1, shapeexp_e1_ivar, shapeexp_e2, shapeexp_e2_ivar, shapedev_r, shapedev_r_ivar, shapedev_e1, shapedev_e1_ivar, shapedev_e2, shapedev_e2_ivar ) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) RETURNING id' +### cursor.execute( query, tuple( [str(elem) for elem in newdata_cand[i]] ) ) id = cursor.fetchone()[0] -## - query = 'INSERT INTO decam ( cand_id, uflux, uflux_ivar, ufracflux, ufracmasked, ufracin, u_rchi2, unobs, u_anymask, u_allmask, u_ext, gflux, gflux_ivar, gfracflux, gfracmasked, gfracin, g_rchi2, gnobs, g_anymask, g_allmask, g_ext, rflux, rflux_ivar, rfracflux, rfracmasked, rfracin, r_rchi2, rnobs, r_anymask, r_allmask, r_ext, iflux, iflux_ivar, ifracflux, ifracmasked, ifracin, i_rchi2, inobs, i_anymask, i_allmask, i_ext, zflux, zflux_ivar, zfracflux, zfracmasked, zfracin, z_rchi2, znobs, z_anymask, z_allmask, z_ext, yflux, yflux_ivar, yfracflux, yfracmasked, yfracin, y_rchi2, ynobs, y_anymask, y_allmask, y_ext) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )' -## +### + query = 'INSERT INTO decam ( cand_id, uflux, uflux_ivar, ufracflux, ufracmasked, ufracin, u_rchi2, unobs, u_anymask, u_allmask, u_psfsize, u_ext, u_depth, u_galdepth, gflux, gflux_ivar, gfracflux, gfracmasked, gfracin, g_rchi2, gnobs, g_anymask, g_allmask, g_psfsize, g_ext, g_depth, g_galdepth, rflux, rflux_ivar, rfracflux, rfracmasked, rfracin, r_rchi2, rnobs, r_anymask, r_allmask, r_psfsize, r_ext, r_depth, r_galdepth, iflux, iflux_ivar, ifracflux, ifracmasked, ifracin, i_rchi2, inobs, i_anymask, i_allmask, i_psfsize, i_ext, i_depth, i_galdepth, zflux, zflux_ivar, zfracflux, zfracmasked, zfracin, z_rchi2, znobs, z_anymask, z_allmask, z_psfsize, z_ext, z_depth, z_galdepth, yflux, yflux_ivar, yfracflux, yfracmasked, yfracin, y_rchi2, ynobs, y_anymask, y_allmask, y_psfsize, y_ext, y_depth, y_galdepth) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )' +### cursor.execute( query, tuple( [str(elem) for elem in [ id ] + newdata_decam[i]] ) ) -## -## +### +### query = 'INSERT INTO wise (cand_id, w1flux, w1flux_ivar, w1fracflux, w1_rchi2, w1nobs, w1_ext, w2flux, w2flux_ivar, w2fracflux, w2_rchi2, w2nobs, w2_ext, w3flux, w3flux_ivar, w3fracflux, w3_rchi2, w3nobs, w3_ext, w4flux, w4flux_ivar, w4fracflux, w4_rchi2, w4nobs, w4_ext ) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )' -## +### cursor.execute( query, tuple( [str(elem) for elem in [ id ] + newdata_wise[i]] ) ) -# ## +### query = 'INSERT INTO decam_aper ( cand_id, uapflux_1, uapflux_2, uapflux_3, uapflux_4, uapflux_5, uapflux_6, uapflux_7, uapflux_8, uapflux_resid_1, uapflux_resid_2, uapflux_resid_3, uapflux_resid_4, uapflux_resid_5, uapflux_resid_6, uapflux_resid_7, uapflux_resid_8, uapflux_ivar_1, uapflux_ivar_2, uapflux_ivar_3, uapflux_ivar_4, uapflux_ivar_5, uapflux_ivar_6, uapflux_ivar_7, uapflux_ivar_8, gapflux_1, gapflux_2, gapflux_3, gapflux_4, gapflux_5, gapflux_6, gapflux_7, gapflux_8, gapflux_resid_1, gapflux_resid_2, gapflux_resid_3, gapflux_resid_4, gapflux_resid_5, gapflux_resid_6, gapflux_resid_7, gapflux_resid_8, gapflux_ivar_1, gapflux_ivar_2, gapflux_ivar_3, gapflux_ivar_4, gapflux_ivar_5, gapflux_ivar_6, gapflux_ivar_7, gapflux_ivar_8, rapflux_1, rapflux_2, rapflux_3, rapflux_4, rapflux_5, rapflux_6, rapflux_7, rapflux_8, rapflux_resid_1, rapflux_resid_2, rapflux_resid_3, rapflux_resid_4, rapflux_resid_5, rapflux_resid_6, rapflux_resid_7, rapflux_resid_8, rapflux_ivar_1, rapflux_ivar_2, rapflux_ivar_3, rapflux_ivar_4, rapflux_ivar_5, rapflux_ivar_6, rapflux_ivar_7, rapflux_ivar_8, iapflux_1, iapflux_2, iapflux_3, iapflux_4, iapflux_5, iapflux_6, iapflux_7, iapflux_8, iapflux_resid_1, iapflux_resid_2, iapflux_resid_3, iapflux_resid_4, iapflux_resid_5, iapflux_resid_6, iapflux_resid_7, iapflux_resid_8, iapflux_ivar_1, iapflux_ivar_2, iapflux_ivar_3, iapflux_ivar_4, iapflux_ivar_5, iapflux_ivar_6, iapflux_ivar_7, iapflux_ivar_8, zapflux_1, zapflux_2, zapflux_3, zapflux_4, zapflux_5, zapflux_6, zapflux_7, zapflux_8, zapflux_resid_1, zapflux_resid_2, zapflux_resid_3, zapflux_resid_4, zapflux_resid_5, zapflux_resid_6, zapflux_resid_7, zapflux_resid_8, zapflux_ivar_1, zapflux_ivar_2, zapflux_ivar_3, zapflux_ivar_4, zapflux_ivar_5, zapflux_ivar_6, zapflux_ivar_7, zapflux_ivar_8, yapflux_1, yapflux_2, yapflux_3, yapflux_4, yapflux_5, yapflux_6, yapflux_7, yapflux_8, yapflux_resid_1, yapflux_resid_2, yapflux_resid_3, yapflux_resid_4, yapflux_resid_5, yapflux_resid_6, yapflux_resid_7, yapflux_resid_8, yapflux_ivar_1, yapflux_ivar_2, yapflux_ivar_3, yapflux_ivar_4, yapflux_ivar_5, yapflux_ivar_6, yapflux_ivar_7, yapflux_ivar_8 ) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )' -## +### cursor.execute( query, tuple( [str(elem) for elem in [ id ] + newdata_decap[i]] ) ) +### +# +## ## - -# -# con.commit() print fimage, 'has been loaded' diff --git a/db/bricks.schema.dr1 b/db/bricks.table.dr2 similarity index 58% rename from db/bricks.schema.dr1 rename to db/bricks.table.dr2 index f42a98059..6fdae949c 100644 --- a/db/bricks.schema.dr1 +++ b/db/bricks.table.dr2 @@ -13,9 +13,12 @@ CREATE TABLE bricks ( dec1 double precision, ra2 double precision, dec2 double precision, - has_g boolean, - has_r boolean, - has_z boolean, + nobs_med_g integer, + nobs_med_r integer, + nobs_med_z integer, + nobs_max_g integer, + nobs_max_r integer, + nobs_max_z integer, filename text default null, tractorvr text default null, loaded boolean default 'false' @@ -24,7 +27,10 @@ CREATE TABLE bricks ( CREATE INDEX bricks_q3c_idx ON bricks (q3c_ang2ipix(ra,dec)); CLUSTER bricks_q3c_idx on bricks; CREATE INDEX bricks_brickid_idx ON bricks (brickid); -CREATE INDEX bricks_hasg_idx ON bricks (has_g); -CREATE INDEX bricks_hasr_idx ON bricks (has_r); -CREATE INDEX bricks_hasz_idx ON bricks (has_z); +CREATE INDEX bricks_medg_idx ON bricks (nobs_med_g); +CREATE INDEX bricks_medr_idx ON bricks (nobs_med_r); +CREATE INDEX bricks_medz_idx ON bricks (nobs_med_z); +CREATE INDEX bricks_maxg_idx ON bricks (nobs_max_g); +CREATE INDEX bricks_maxr_idx ON bricks (nobs_max_r); +CREATE INDEX bricks_maxz_idx ON bricks (nobs_max_z); diff --git a/db/candidate.schema.dr1 b/db/candidate.table.dr2 similarity index 83% rename from db/candidate.schema.dr1 rename to db/candidate.table.dr2 index 29fcd0304..72ead9b89 100644 --- a/db/candidate.schema.dr1 +++ b/db/candidate.table.dr2 @@ -5,6 +5,8 @@ CREATE TABLE candidate ( brickid integer, objid integer, blob integer, + ninblob integer, + tycho2inblob boolean, "type" text, ra double precision, ra_ivar real, @@ -21,6 +23,7 @@ CREATE TABLE candidate ( DCHISQ2 real, DCHISQ3 real, DCHISQ4 real, + DCHISQ5 real, FRACDEV real, FRACDEV_IVAR real, SHAPEEXP_R real, @@ -37,6 +40,3 @@ CREATE TABLE candidate ( SHAPEDEV_E2_IVAR real ); -CREATE INDEX cand_q3c_candidate_idx ON candidate (q3c_ang2ipix(ra,dec)); -CLUSTER cand_q3c_candidate_idx on candidate; -CREATE INDEX cand_brickid_idx ON candidate (brickid); diff --git a/db/decam.dr1 b/db/decam.table.dr2 similarity index 78% rename from db/decam.dr1 rename to db/decam.table.dr2 index 91f6be9f4..1f758c6a0 100644 --- a/db/decam.dr1 +++ b/db/decam.table.dr2 @@ -12,7 +12,10 @@ CREATE TABLE decam ( unobs integer, u_anymask integer, u_allmask integer, + u_psfsize real, u_ext real, + u_depth real, + u_galdepth real, gflux real, gflux_ivar real, gfracflux real, @@ -22,7 +25,10 @@ CREATE TABLE decam ( gnobs integer, g_anymask integer, g_allmask integer, + g_psfsize real, g_ext real, + g_depth real, + g_galdepth real, rflux real, rflux_ivar real, rfracflux real, @@ -32,7 +38,10 @@ CREATE TABLE decam ( rnobs integer, r_anymask integer, r_allmask integer, + r_psfsize real, r_ext real, + r_depth real, + r_galdepth real, iflux real, iflux_ivar real, ifracflux real, @@ -42,7 +51,10 @@ CREATE TABLE decam ( inobs integer, i_anymask integer, i_allmask integer, + i_psfsize real, i_ext real, + i_depth real, + i_galdepth real, zflux real, zflux_ivar real, zfracflux real, @@ -52,7 +64,10 @@ CREATE TABLE decam ( znobs integer, z_anymask integer, z_allmask integer, + z_psfsize real, z_ext real, + z_depth real, + z_galdepth real, Yflux real, Yflux_ivar real, Yfracflux real, @@ -62,7 +77,9 @@ CREATE TABLE decam ( Ynobs integer, Y_anymask integer, Y_allmask integer, - Y_ext real + Y_psfsize real, + Y_ext real, + Y_depth real, + Y_galdepth real ); -CREATE INDEX decam_candid_idx ON decam (cand_id); diff --git a/db/decam_aper.dr1 b/db/decam_aper.table.dr2 similarity index 98% rename from db/decam_aper.dr1 rename to db/decam_aper.table.dr2 index 75bb27b9e..b9df5aac1 100644 --- a/db/decam_aper.dr1 +++ b/db/decam_aper.table.dr2 @@ -149,4 +149,3 @@ CREATE TABLE decam_aper ( yapflux_ivar_8 real ); -CREATE INDEX decam_aper_candid_idx ON decam_aper (cand_id); diff --git a/db/images.schema.dr1 b/db/images.table.dr2 similarity index 100% rename from db/images.schema.dr1 rename to db/images.table.dr2 diff --git a/db/index.dr1 b/db/index.table.dr2 similarity index 77% rename from db/index.dr1 rename to db/index.table.dr2 index fd1b54910..11367f762 100644 --- a/db/index.dr1 +++ b/db/index.table.dr2 @@ -1,3 +1,9 @@ +CREATE INDEX cand_q3c_candidate_idx ON candidate (q3c_ang2ipix(ra,dec)); +CLUSTER cand_q3c_candidate_idx on candidate; +CREATE INDEX cand_brickid_idx ON candidate (brickid); +CREATE INDEX decam_candid_idx ON decam (cand_id); +CREATE INDEX decam_aper_candid_idx ON decam_aper (cand_id); +CREATE INDEX wise_candid_idx ON wise (cand_id); CREATE INDEX uflux_idx ON decam (uflux); CREATE INDEX gflux_idx ON decam (gflux); CREATE INDEX rflux_idx ON decam (rflux); diff --git a/db/ps1.schema.dr1 b/db/ps1.table.dr2 similarity index 100% rename from db/ps1.schema.dr1 rename to db/ps1.table.dr2 diff --git a/db/unittest_successful_load.py b/db/unittest_successful_load.py new file mode 100644 index 000000000..74c11c28e --- /dev/null +++ b/db/unittest_successful_load.py @@ -0,0 +1,50 @@ +import psycopg2 +import argparse +import astropy +from astropy.io import fits + +import sys, os, re, glob, distutils +from distutils.util import strtobool +import numpy as np + +parser = argparse.ArgumentParser(description="test") +parser.add_argument("--tractor_catalog",action="store",help='fits table',default='/project/projectdirs/desiproc/dr2/tractor/000/tractor-0006p035.fits',required=False) +args = parser.parse_args() + +print "args.tractor_catalog= ",args.tractor_catalog + +fitsbin = args.tractor_catalog +table = fits.open( fitsbin ) +# Access the data part of the table. +trac = table[1].data +trac = np.asarray(trac) +ans={} +i=1000 +ans['cand']=trac['blob'][i],trac['ra'][i],trac['dec'][i],trac['dchisq'][i][1],trac['dchisq'][i][3] +ans['decam']=trac['decam_nobs'][i][4],trac['decam_flux'][i][1],trac['decam_flux'][i][2],trac['decam_flux'][i][4] +ans['decam_aper']=trac['decam_apflux'][i][1][0],trac['decam_apflux'][i][2][0],trac['decam_apflux'][i][4][0],trac['decam_apflux'][i][4][7] +ans['wise']=trac['wise_flux'][i][0],trac['wise_flux'][i][1],trac['wise_flux'][i][2],trac['wise_flux'][i][3],trac['wise_flux_ivar'][i][3] + +#db +con = psycopg2.connect(host='scidb2.nersc.gov', user='desi_user', database='desi') +cur = con.cursor() +def output(cursor,query): + cursor.execute(query) + return cursor.fetchall() + +sql={} +sql['cand']= output(cur,"SELECT blob,ra,dec,dchisq2,dchisq4,id from dr2.candidate where objid=1000") +sql['decam']= output(cur,"SELECT gnobs,gflux,rflux,zflux from dr2.decam where cand_id=884") +sql['decam_aper']= output(cur,"SELECT gapflux_1,rapflux_1,zapflux_1,zapflux_8 from dr2.decam_aper where cand_id=884") +sql['wise']= output(cur,"SELECT w1flux,w2flux,w3flux,w4flux,w4flux_ivar from dr2.wise where cand_id=884") + +for key in ans.keys(): + print '----',key.upper(),'----' + print 'tractor cat: ',ans[key] + print 'postgres : ',sql[key][0] + +# Nothing should be commited to the db since this is a test, so +# LEAVE THE FOLLOWING COMMENTED OUT: +#con.commit() +print 'done' + diff --git a/db/wise.dr1 b/db/wise.table.dr2 similarity index 93% rename from db/wise.dr1 rename to db/wise.table.dr2 index 1156c527b..056847be0 100644 --- a/db/wise.dr1 +++ b/db/wise.table.dr2 @@ -29,4 +29,3 @@ CREATE TABLE wise ( W4_ext real ); -CREATE INDEX wise_candid_idx ON wise (cand_id); diff --git a/db/zp_schema.dr1 b/db/zp_schema.table.dr2 similarity index 100% rename from db/zp_schema.dr1 rename to db/zp_schema.table.dr2