Merge pull request #53 from desihub/dr2

DR2 Tractor Catalogues psql tables and load scripts

bin/bricks_load.py

@@ -1,9 +1,9 @@
 #----------------------------------------------------------------------#
 # filename: pyfits+psycopg2_ex.py 
 # author: Peter Nugent
-# date: 10/30/2014
+# date: 1/29/2016
 # ---------------------------------------------------------------------#
-# Function: Read in a Arjun fits binary table from standard in and load it
+# Function: DR2, Read in a Arjun fits binary table from standard in and load it
 # into the desi calib pg table database with psycopg2.
 # ---------------------------------------------------------------------#
 
@@ -43,43 +43,28 @@
 
 # Fire up the db
 
-con = psycopg2.connect(host='scidb2.nersc.gov', user='desi_admin', password='L00cy-1959', database='desi')
+con = psycopg2.connect(host='scidb2.nersc.gov', user='desi_admin', database='desi')
 cursor = con.cursor()
 
 # Re-cast as strings so the load is easy 
 
 for i in range(0, nrows):
 
-    if tbdata['has_g'][i] == 70:
-        g = 0
-    else:
-	g = 1
-
-    if tbdata['has_r'][i] == 70:
-        r = 0
-    else:
-	r = 1
-
-    if tbdata['has_z'][i] == 70:
-        z = 0
-    else:
-	z = 1
-
-    line = [ tbdata['brickname'][i], tbdata['brickid'][i], tbdata['brickrow'][i], tbdata['brickcol'][i], tbdata['brickq'][i], tbdata['ra'][i], tbdata['dec'][i], tbdata['ra1'][i], tbdata['dec1'][i], tbdata['ra2'][i], tbdata['dec2'][i], bool(g), bool(r), bool(z) ]
+    line = [ tbdata['brickname'][i], tbdata['brickid'][i], tbdata['brickrow'][i], tbdata['brickcol'][i], tbdata['brickq'][i], tbdata['ra'][i], tbdata['dec'][i], tbdata['ra1'][i], tbdata['dec1'][i], tbdata['ra2'][i], tbdata['dec2'][i], tbdata['nobs_med_g'][i], tbdata['nobs_med_r'][i], tbdata['nobs_med_z'][i], tbdata['nobs_max_g'][i], tbdata['nobs_max_r'][i], tbdata['nobs_max_z'][i] ]
 
     newdata.append(line) 
 
 ## Re-cast as strings so the load is easy 
 #
 for i, f in enumerate(newdata):
 ##
-   query = 'INSERT INTO bricks ( brickname, brickid, brickrow, brickcol, brickq, ra, dec, ra1, dec1, ra2, dec2, has_g, has_r, has_z) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )' 
+   query = 'INSERT INTO bricks ( brickname, brickid, brickrow, brickcol, brickq, ra, dec, ra1, dec1, ra2, dec2, nobs_med_g, nobs_med_r, nobs_med_z, nobs_max_g, nobs_max_r, nobs_max_z) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )' 
 ##
    cursor.execute( query, tuple( [str(elem) for elem in newdata[i]] ) ) 
 #
 #
 
 con.commit()
-
+print 'done'
 # That's it!
 

db/bricks.schema.dr1 → db/bricks.table.dr2

@@ -13,9 +13,12 @@ CREATE TABLE bricks (
     dec1 double precision,
     ra2 double precision,
     dec2 double precision,
-    has_g boolean,
-    has_r boolean,
-    has_z boolean,
+    nobs_med_g integer,
+    nobs_med_r integer,
+    nobs_med_z integer,
+    nobs_max_g integer,
+    nobs_max_r integer,
+    nobs_max_z integer,
     filename text default null,
     tractorvr text default null,
     loaded boolean default 'false'
@@ -24,7 +27,10 @@ CREATE TABLE bricks (
 CREATE INDEX bricks_q3c_idx ON bricks (q3c_ang2ipix(ra,dec));
 CLUSTER bricks_q3c_idx on bricks;
 CREATE INDEX bricks_brickid_idx ON bricks (brickid);
-CREATE INDEX bricks_hasg_idx ON bricks (has_g);
-CREATE INDEX bricks_hasr_idx ON bricks (has_r);
-CREATE INDEX bricks_hasz_idx ON bricks (has_z);
+CREATE INDEX bricks_medg_idx ON bricks (nobs_med_g);
+CREATE INDEX bricks_medr_idx ON bricks (nobs_med_r);
+CREATE INDEX bricks_medz_idx ON bricks (nobs_med_z);
+CREATE INDEX bricks_maxg_idx ON bricks (nobs_max_g);
+CREATE INDEX bricks_maxr_idx ON bricks (nobs_max_r);
+CREATE INDEX bricks_maxz_idx ON bricks (nobs_max_z);
 

db/candidate.schema.dr1 → db/candidate.table.dr2

@@ -5,6 +5,8 @@ CREATE TABLE candidate (
     brickid integer,
     objid integer,
     blob integer,
+    ninblob integer,
+    tycho2inblob boolean,
     "type" text,
     ra double precision,
     ra_ivar real,
@@ -21,6 +23,7 @@ CREATE TABLE candidate (
     DCHISQ2 real,
     DCHISQ3 real,
     DCHISQ4 real,
+    DCHISQ5 real,
     FRACDEV real,
     FRACDEV_IVAR real,
     SHAPEEXP_R real,
@@ -37,6 +40,3 @@ CREATE TABLE candidate (
     SHAPEDEV_E2_IVAR real
 );
 
-CREATE INDEX cand_q3c_candidate_idx ON candidate (q3c_ang2ipix(ra,dec));
-CLUSTER cand_q3c_candidate_idx on candidate;
-CREATE INDEX cand_brickid_idx ON candidate (brickid);

db/decam.dr1 → db/decam.table.dr2

@@ -12,7 +12,10 @@ CREATE TABLE decam (
     unobs integer,
     u_anymask integer,
     u_allmask integer,
+    u_psfsize real,
     u_ext real,
+    u_depth real,
+    u_galdepth real,
     gflux real,
     gflux_ivar real,
     gfracflux real,
@@ -22,7 +25,10 @@ CREATE TABLE decam (
     gnobs integer,
     g_anymask integer,
     g_allmask integer,
+    g_psfsize real,
     g_ext real,
+    g_depth real,
+    g_galdepth real,
     rflux real,
     rflux_ivar real,
     rfracflux real,
@@ -32,7 +38,10 @@ CREATE TABLE decam (
     rnobs integer,
     r_anymask integer,
     r_allmask integer,
+    r_psfsize real,
     r_ext real,
+    r_depth real,
+    r_galdepth real,
     iflux real,
     iflux_ivar real,
     ifracflux real,
@@ -42,7 +51,10 @@ CREATE TABLE decam (
     inobs integer,
     i_anymask integer,
     i_allmask integer,
+    i_psfsize real,
     i_ext real,
+    i_depth real,
+    i_galdepth real,
     zflux real,
     zflux_ivar real,
     zfracflux real,
@@ -52,7 +64,10 @@ CREATE TABLE decam (
     znobs integer,
     z_anymask integer,
     z_allmask integer,
+    z_psfsize real,
     z_ext real,
+    z_depth real,
+    z_galdepth real,
     Yflux real,
     Yflux_ivar real,
     Yfracflux real,
@@ -62,7 +77,9 @@ CREATE TABLE decam (
     Ynobs integer,
     Y_anymask integer,
     Y_allmask integer,
-    Y_ext real
+    Y_psfsize real,
+    Y_ext real,
+    Y_depth real,
+    Y_galdepth real
 );
 
-CREATE INDEX decam_candid_idx ON decam (cand_id);

db/decam_aper.dr1 → db/decam_aper.table.dr2

@@ -149,4 +149,3 @@ CREATE TABLE decam_aper (
     yapflux_ivar_8 real
 );
 
-CREATE INDEX decam_aper_candid_idx ON decam_aper (cand_id);

db/index.dr1 → db/index.table.dr2

@@ -1,3 +1,9 @@
+CREATE INDEX cand_q3c_candidate_idx ON candidate (q3c_ang2ipix(ra,dec));
+CLUSTER cand_q3c_candidate_idx on candidate;
+CREATE INDEX cand_brickid_idx ON candidate (brickid);
+CREATE INDEX decam_candid_idx ON decam (cand_id);
+CREATE INDEX decam_aper_candid_idx ON decam_aper (cand_id);
+CREATE INDEX wise_candid_idx ON wise (cand_id);
 CREATE INDEX uflux_idx ON decam (uflux);
 CREATE INDEX gflux_idx ON decam (gflux);
 CREATE INDEX rflux_idx ON decam (rflux);

db/unittest_successful_load.py

@@ -0,0 +1,50 @@
+import psycopg2 
+import argparse
+import astropy
+from astropy.io import fits
+
+import sys, os, re, glob, distutils
+from distutils.util import strtobool
+import numpy as np
+
+parser = argparse.ArgumentParser(description="test")
+parser.add_argument("--tractor_catalog",action="store",help='fits table',default='/project/projectdirs/desiproc/dr2/tractor/000/tractor-0006p035.fits',required=False)
+args = parser.parse_args()
+
+print "args.tractor_catalog= ",args.tractor_catalog
+
+fitsbin = args.tractor_catalog
+table = fits.open( fitsbin )
+# Access the data part of the table.
+trac = table[1].data
+trac = np.asarray(trac)
+ans={}
+i=1000
+ans['cand']=trac['blob'][i],trac['ra'][i],trac['dec'][i],trac['dchisq'][i][1],trac['dchisq'][i][3]
+ans['decam']=trac['decam_nobs'][i][4],trac['decam_flux'][i][1],trac['decam_flux'][i][2],trac['decam_flux'][i][4]
+ans['decam_aper']=trac['decam_apflux'][i][1][0],trac['decam_apflux'][i][2][0],trac['decam_apflux'][i][4][0],trac['decam_apflux'][i][4][7]
+ans['wise']=trac['wise_flux'][i][0],trac['wise_flux'][i][1],trac['wise_flux'][i][2],trac['wise_flux'][i][3],trac['wise_flux_ivar'][i][3]
+
+#db
+con = psycopg2.connect(host='scidb2.nersc.gov', user='desi_user', database='desi')
+cur = con.cursor()
+def output(cursor,query):
+    cursor.execute(query)
+    return cursor.fetchall()
+
+sql={}
+sql['cand']= output(cur,"SELECT blob,ra,dec,dchisq2,dchisq4,id from dr2.candidate where objid=1000")
+sql['decam']= output(cur,"SELECT gnobs,gflux,rflux,zflux from dr2.decam where cand_id=884")
+sql['decam_aper']= output(cur,"SELECT gapflux_1,rapflux_1,zapflux_1,zapflux_8 from dr2.decam_aper where cand_id=884")
+sql['wise']= output(cur,"SELECT w1flux,w2flux,w3flux,w4flux,w4flux_ivar from dr2.wise where cand_id=884")
+
+for key in ans.keys():
+    print '----',key.upper(),'----'
+    print 'tractor cat: ',ans[key]
+    print 'postgres   : ',sql[key][0]
+
+# Nothing should be commited to the db since this is a test, so
+# LEAVE THE FOLLOWING COMMENTED OUT: 
+#con.commit()
+print 'done'
+

db/wise.dr1 → db/wise.table.dr2

@@ -29,4 +29,3 @@ CREATE TABLE wise (
     W4_ext real
 );
 
-CREATE INDEX wise_candid_idx ON wise (cand_id);