diff --git a/api/handlers.go b/api/handlers.go index d7cd28310..5488ca5f7 100644 --- a/api/handlers.go +++ b/api/handlers.go @@ -27,12 +27,15 @@ import ( // * "dbname" is the name of the database func branchesHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "branches", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -102,6 +105,9 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "columns", r.Header.Get("User-Agent")) + // Extract the table name table, err := com.GetFormTable(r, false) if err != nil { @@ -221,12 +227,15 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database func commitsHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "commits", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -271,6 +280,9 @@ func databasesHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, "", "", "databases", r.Header.Get("User-Agent")) + // Get "live" boolean value, if provided by the caller var live bool live, err = com.GetFormLive(r) @@ -336,6 +348,9 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) { } dbOwner := loggedInUser + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "delete", r.Header.Get("User-Agent")) + // Check if the database exists exists, err := com.CheckDBPermissions(loggedInUser, dbOwner, dbName, false) if err != nil { @@ -517,6 +532,10 @@ func diffHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + // Note - Lets not bother adding additional api logging fields just for the diff function at this stage + com.ApiCallLog(loggedInUser, dbOwnerA, dbNameA, "diff", r.Header.Get("User-Agent")) + // Check permissions of the first database var allowed bool allowed, err = com.CheckDBPermissions(loggedInUser, dbOwnerA, dbNameA, false) @@ -593,6 +612,9 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "download", r.Header.Get("User-Agent")) + // Return the requested database to the user _, err = com.DownloadDatabase(w, r, dbOwner, dbName, commitID, loggedInUser, "api") if err != nil { @@ -630,6 +652,9 @@ func executeHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "execute", r.Header.Get("User-Agent")) + // Grab the incoming SQLite query rawInput := r.FormValue("sql") var sql string @@ -709,6 +734,9 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "indexes", r.Header.Get("User-Agent")) + // Check if the database is a live database, and get the node/queue to send the request to isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -826,12 +854,15 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database func metadataHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "metadata", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -886,6 +917,9 @@ func queryHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "query", r.Header.Get("User-Agent")) + // Grab the incoming SQLite query rawInput := r.FormValue("sql") query, err := com.CheckUnicode(rawInput) @@ -958,12 +992,15 @@ func queryHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database func releasesHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "releases", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -1033,6 +1070,9 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "tables", r.Header.Get("User-Agent")) + // Check if the database is a live database, and get the node/queue to send the request to isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -1108,12 +1148,15 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database func tagsHandler(w http.ResponseWriter, r *http.Request) { // Do auth check, grab request info - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "tags", r.Header.Get("User-Agent")) + // If the database is a live database, we return an error message isLive, _, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -1263,6 +1306,9 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, loggedInUser, dbName, "upload", r.Header.Get("User-Agent")) + // Check if the database exists already exists, err := com.CheckDBExists(loggedInUser, dbName) if err != nil { @@ -1366,6 +1412,9 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) { return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, loggedInUser, dbName, "views", r.Header.Get("User-Agent")) + // Check if the database is a live database, and get the node/queue to send the request to isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName) if err != nil { @@ -1441,12 +1490,15 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) { // * "dbname" is the name of the database being queried func webpageHandler(w http.ResponseWriter, r *http.Request) { // Authenticate user and collect requested database details - _, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) + loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r) if err != nil { jsonErr(w, err.Error(), httpStatus) return } + // Record the api call in our backend database + com.ApiCallLog(loggedInUser, dbOwner, dbName, "views", r.Header.Get("User-Agent")) + // Return the database webUI URL to the user var z com.WebpageResponseContainer z.WebPage = "https://" + com.Conf.Web.ServerName + "/" + dbOwner + "/" + dbName diff --git a/api/main.go b/api/main.go index 11c305669..d7ffa4ee4 100644 --- a/api/main.go +++ b/api/main.go @@ -3,10 +3,6 @@ package main // TODO: API functions that still need updating for Live databases // * diff - already updated to just return an error for live databases. needs testing though -// FIXME: Update the documented Upload() function return values on the API doc page. Currently it talks about -// returning the commit ID for the upload. We'll probably return that field with a blank value for live -// databases though. TBD. - // FIXME: After the API and webui pieces are done, figure out how the DB4S end // point and dio should be updated to use live databases too @@ -113,29 +109,29 @@ func main() { } // Our pages - http.Handle("/", gz.GzipHandler(handleWrapper(rootHandler))) - http.Handle("/changelog", gz.GzipHandler(handleWrapper(changeLogHandler))) - http.Handle("/changelog.html", gz.GzipHandler(handleWrapper(changeLogHandler))) - http.Handle("/v1/branches", gz.GzipHandler(handleWrapper(branchesHandler))) - http.Handle("/v1/columns", gz.GzipHandler(handleWrapper(columnsHandler))) - http.Handle("/v1/commits", gz.GzipHandler(handleWrapper(commitsHandler))) - http.Handle("/v1/databases", gz.GzipHandler(handleWrapper(databasesHandler))) - http.Handle("/v1/delete", gz.GzipHandler(handleWrapper(deleteHandler))) - http.Handle("/v1/diff", gz.GzipHandler(handleWrapper(diffHandler))) - http.Handle("/v1/download", gz.GzipHandler(handleWrapper(downloadHandler))) - http.Handle("/v1/execute", gz.GzipHandler(handleWrapper(executeHandler))) - http.Handle("/v1/indexes", gz.GzipHandler(handleWrapper(indexesHandler))) - http.Handle("/v1/metadata", gz.GzipHandler(handleWrapper(metadataHandler))) - http.Handle("/v1/query", gz.GzipHandler(handleWrapper(queryHandler))) - http.Handle("/v1/releases", gz.GzipHandler(handleWrapper(releasesHandler))) - http.Handle("/v1/tables", gz.GzipHandler(handleWrapper(tablesHandler))) - http.Handle("/v1/tags", gz.GzipHandler(handleWrapper(tagsHandler))) - http.Handle("/v1/upload", gz.GzipHandler(handleWrapper(uploadHandler))) - http.Handle("/v1/views", gz.GzipHandler(handleWrapper(viewsHandler))) - http.Handle("/v1/webpage", gz.GzipHandler(handleWrapper(webpageHandler))) + http.Handle("/", gz.GzipHandler(corsWrapper(rootHandler))) + http.Handle("/changelog", gz.GzipHandler(corsWrapper(changeLogHandler))) + http.Handle("/changelog.html", gz.GzipHandler(corsWrapper(changeLogHandler))) + http.Handle("/v1/branches", gz.GzipHandler(corsWrapper(branchesHandler))) + http.Handle("/v1/columns", gz.GzipHandler(corsWrapper(columnsHandler))) + http.Handle("/v1/commits", gz.GzipHandler(corsWrapper(commitsHandler))) + http.Handle("/v1/databases", gz.GzipHandler(corsWrapper(databasesHandler))) + http.Handle("/v1/delete", gz.GzipHandler(corsWrapper(deleteHandler))) + http.Handle("/v1/diff", gz.GzipHandler(corsWrapper(diffHandler))) + http.Handle("/v1/download", gz.GzipHandler(corsWrapper(downloadHandler))) + http.Handle("/v1/execute", gz.GzipHandler(corsWrapper(executeHandler))) + http.Handle("/v1/indexes", gz.GzipHandler(corsWrapper(indexesHandler))) + http.Handle("/v1/metadata", gz.GzipHandler(corsWrapper(metadataHandler))) + http.Handle("/v1/query", gz.GzipHandler(corsWrapper(queryHandler))) + http.Handle("/v1/releases", gz.GzipHandler(corsWrapper(releasesHandler))) + http.Handle("/v1/tables", gz.GzipHandler(corsWrapper(tablesHandler))) + http.Handle("/v1/tags", gz.GzipHandler(corsWrapper(tagsHandler))) + http.Handle("/v1/upload", gz.GzipHandler(corsWrapper(uploadHandler))) + http.Handle("/v1/views", gz.GzipHandler(corsWrapper(viewsHandler))) + http.Handle("/v1/webpage", gz.GzipHandler(corsWrapper(webpageHandler))) // favicon.ico - http.Handle("/favicon.ico", gz.GzipHandler(handleWrapper(func(w http.ResponseWriter, r *http.Request) { + http.Handle("/favicon.ico", gz.GzipHandler(corsWrapper(func(w http.ResponseWriter, r *http.Request) { logReq(r, "-") http.ServeFile(w, r, filepath.Join(com.Conf.Web.BaseDir, "webui", "favicon.ico")) }))) @@ -329,9 +325,8 @@ func extractUserFromClientCert(w http.ResponseWriter, r *http.Request) (userAcc return } -// handleWrapper does nothing useful except interface between types -// TODO: Get rid of this, as it shouldn't be needed -func handleWrapper(fn http.HandlerFunc) http.HandlerFunc { +// corsWrapper sets a general allow for all our api calls +func corsWrapper(fn http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Enable CORS (https://enable-cors.org) w.Header().Set("Access-Control-Allow-Origin", "*") @@ -361,6 +356,5 @@ func jsonErr(w http.ResponseWriter, msg string, statusCode int) { // logReq writes an entry for the incoming request to the request log func logReq(r *http.Request, loggedInUser string) { fmt.Fprintf(reqLog, "%v - %s [%s] \"%s %s %s\" \"-\" \"-\" \"%s\" \"%s\"\n", r.RemoteAddr, - loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto, - r.Referer(), r.Header.Get("User-Agent")) + loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto, r.Referer(), r.Header.Get("User-Agent")) } diff --git a/common/postgresql.go b/common/postgresql.go index b9f9ccb97..f45fa9335 100644 --- a/common/postgresql.go +++ b/common/postgresql.go @@ -96,6 +96,51 @@ func AddUser(auth0ID, userName, password, email, displayName, avatarURL string) return nil } +// ApiCallLog records an API call operation. Database name is optional, as not all API calls operate on a +// database. If a database name is provided however, then the database owner name *must* also be provided +func ApiCallLog(loggedInUser, dbOwner, dbName, operation, callerSw string) { + var dbQuery string + var err error + var commandTag pgx.CommandTag + if dbName != "" { + dbQuery = ` + WITH loggedIn AS ( + SELECT user_id + FROM users + WHERE lower(user_name) = lower($1) + ), + WITH owner AS ( + SELECT user_id + FROM users + WHERE lower(user_name) = lower($2) + ), d AS ( + SELECT db.db_id + FROM sqlite_databases AS db, owner + WHERE db.user_id = owner.user_id + AND db.db_name = $3) + INSERT INTO api_call_log (caller_id, db_owner_id, db_id, api_operation, api_caller_sw) + VALUES ((SELECT user_id FROM loggedIn), (SELECT user_id FROM owner), (SELECT db_id FROM d), $4, $5)` + commandTag, err = pdb.Exec(dbQuery, loggedInUser, dbOwner, dbName, operation, callerSw) + } else { + dbQuery = ` + WITH loggedIn AS ( + SELECT user_id + FROM users + WHERE lower(user_name) = lower($1) + ) + INSERT INTO api_call_log (caller_id, api_operation, api_caller_sw) + VALUES ((SELECT user_id FROM loggedIn), $2, $3)` + commandTag, err = pdb.Exec(dbQuery, loggedInUser, operation, callerSw) + } + if err != nil { + log.Printf("Adding api call log entry failed: %s", err) + return + } + if numRows := commandTag.RowsAffected(); numRows != 1 { + log.Printf("Wrong number of rows (%d) affected when adding api call entry for user '%s'", numRows, SanitiseLogString(loggedInUser)) + } +} + // APIKeySave saves a new API key to the PostgreSQL database func APIKeySave(key, loggedInUser string, dateCreated time.Time) error { // Make sure the API key isn't already in the database diff --git a/database/dbhub.sql b/database/dbhub.sql index 7629a7ad9..e140aba17 100644 --- a/database/dbhub.sql +++ b/database/dbhub.sql @@ -30,6 +30,28 @@ SET default_tablespace = ''; SET default_table_access_method = heap; +-- +-- Name: api_call_log; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.api_call_log ( + api_call_id bigint NOT NULL, + api_call_date timestamp with time zone DEFAULT now(), + caller_id bigint, + db_owner_id bigint, + db_id bigint, + api_operation text NOT NULL, + api_caller_sw text +); + + +-- +-- Name: COLUMN api_call_log.db_id; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.api_call_log.db_id IS 'This field must be nullable, as not all api calls act on a database'; + + -- -- Name: api_keys; Type: TABLE; Schema: public; Owner: - -- @@ -61,6 +83,25 @@ CREATE SEQUENCE public.api_keys_key_id_seq ALTER SEQUENCE public.api_keys_key_id_seq OWNED BY public.api_keys.key_id; +-- +-- Name: api_log_log_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.api_log_log_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: api_log_log_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.api_log_log_id_seq OWNED BY public.api_call_log.api_call_id; + + -- -- Name: database_downloads; Type: TABLE; Schema: public; Owner: - -- @@ -596,6 +637,13 @@ CREATE TABLE public.watchers ( ); +-- +-- Name: api_call_log api_call_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.api_call_log ALTER COLUMN api_call_id SET DEFAULT nextval('public.api_log_log_id_seq'::regclass); + + -- -- Name: api_keys key_id; Type: DEFAULT; Schema: public; Owner: - -- @@ -1003,6 +1051,14 @@ ALTER TABLE ONLY public.api_keys ADD CONSTRAINT api_keys_users_user_id_fk FOREIGN KEY (user_id) REFERENCES public.users(user_id) ON UPDATE CASCADE ON DELETE SET NULL; +-- +-- Name: api_call_log api_log_users_user_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.api_call_log + ADD CONSTRAINT api_log_users_user_id_fk FOREIGN KEY (caller_id) REFERENCES public.users(user_id); + + -- -- Name: database_downloads database_downloads_db_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --