Skip to content

Commit

Permalink
api, common. Log all API calls into our database, for stats generation
Browse files Browse the repository at this point in the history 
With this commit, we record basic info for each API call so we can
start doing useful statistic generation for API usage.

The backend database needs a new table created to hold the info:

  CREATE TABLE public.api_call_log (
      api_call_id bigint NOT NULL,
      api_call_date timestamp with time zone DEFAULT now(),
      caller_id bigint,
      db_owner_id bigint,
      db_id bigint,
      api_operation text NOT NULL,
      api_caller_sw text
  );

  COMMENT ON COLUMN public.api_call_log.db_owner_id IS 'This field must be nullable, as not all api calls act on a database';
  COMMENT ON COLUMN public.api_call_log.db_id IS 'This field must be nullable, as not all api calls act on a database';

  CREATE SEQUENCE public.api_log_log_id_seq
      START WITH 1
      INCREMENT BY 1
      NO MINVALUE
      NO MAXVALUE
      CACHE 1;

  ALTER SEQUENCE public.api_log_log_id_seq OWNED BY public.api_call_log.api_call_id;

  ALTER TABLE ONLY public.api_call_log ALTER COLUMN api_call_id SET DEFAULT nextval('public.api_log_log_id_seq'::regclass);

  ALTER TABLE ONLY public.api_call_log ADD CONSTRAINT api_log_users_user_id_fk FOREIGN KEY (caller_id) REFERENCES public.users(user_id);
  • Loading branch information
@justinclift
justinclift committed Apr 27, 2023
1 parent 5c78e52 commit 2364188
Show file tree
Hide file tree
Showing 4 changed files with 183 additions and 36 deletions.
64 changes: 58 additions & 6 deletions api/handlers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,15 @@ import (
// * "dbname" is the name of the database
func branchesHandler(w http.ResponseWriter, r *http.Request) {
// Do auth check, grab request info
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
if err != nil {
jsonErr(w, err.Error(), httpStatus)
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "branches", r.Header.Get("User-Agent"))

// If the database is a live database, we return an error message
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
if err != nil {
Expand Down Expand Up @@ -102,6 +105,9 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "columns", r.Header.Get("User-Agent"))

// Extract the table name
table, err := com.GetFormTable(r, false)
if err != nil {
Expand Down Expand Up @@ -221,12 +227,15 @@ func columnsHandler(w http.ResponseWriter, r *http.Request) {
// * "dbname" is the name of the database
func commitsHandler(w http.ResponseWriter, r *http.Request) {
// Do auth check, grab request info
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
if err != nil {
jsonErr(w, err.Error(), httpStatus)
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "commits", r.Header.Get("User-Agent"))

// If the database is a live database, we return an error message
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
if err != nil {
Expand Down Expand Up @@ -271,6 +280,9 @@ func databasesHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, "", "", "databases", r.Header.Get("User-Agent"))

// Get "live" boolean value, if provided by the caller
var live bool
live, err = com.GetFormLive(r)
Expand Down Expand Up @@ -336,6 +348,9 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
}
dbOwner := loggedInUser

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "delete", r.Header.Get("User-Agent"))

// Check if the database exists
exists, err := com.CheckDBPermissions(loggedInUser, dbOwner, dbName, false)
if err != nil {
Expand Down Expand Up @@ -517,6 +532,10 @@ func diffHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
// Note - Lets not bother adding additional api logging fields just for the diff function at this stage
com.ApiCallLog(loggedInUser, dbOwnerA, dbNameA, "diff", r.Header.Get("User-Agent"))

// Check permissions of the first database
var allowed bool
allowed, err = com.CheckDBPermissions(loggedInUser, dbOwnerA, dbNameA, false)
Expand Down Expand Up @@ -593,6 +612,9 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "download", r.Header.Get("User-Agent"))

// Return the requested database to the user
_, err = com.DownloadDatabase(w, r, dbOwner, dbName, commitID, loggedInUser, "api")
if err != nil {
Expand Down Expand Up @@ -630,6 +652,9 @@ func executeHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "execute", r.Header.Get("User-Agent"))

// Grab the incoming SQLite query
rawInput := r.FormValue("sql")
var sql string
Expand Down Expand Up @@ -709,6 +734,9 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "indexes", r.Header.Get("User-Agent"))

// Check if the database is a live database, and get the node/queue to send the request to
isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName)
if err != nil {
Expand Down Expand Up @@ -826,12 +854,15 @@ func indexesHandler(w http.ResponseWriter, r *http.Request) {
// * "dbname" is the name of the database
func metadataHandler(w http.ResponseWriter, r *http.Request) {
// Do auth check, grab request info
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
if err != nil {
jsonErr(w, err.Error(), httpStatus)
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "metadata", r.Header.Get("User-Agent"))

// If the database is a live database, we return an error message
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
if err != nil {
Expand Down Expand Up @@ -886,6 +917,9 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "query", r.Header.Get("User-Agent"))

// Grab the incoming SQLite query
rawInput := r.FormValue("sql")
query, err := com.CheckUnicode(rawInput)
Expand Down Expand Up @@ -958,12 +992,15 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
// * "dbname" is the name of the database
func releasesHandler(w http.ResponseWriter, r *http.Request) {
// Do auth check, grab request info
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
if err != nil {
jsonErr(w, err.Error(), httpStatus)
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "releases", r.Header.Get("User-Agent"))

// If the database is a live database, we return an error message
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
if err != nil {
Expand Down Expand Up @@ -1033,6 +1070,9 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "tables", r.Header.Get("User-Agent"))

// Check if the database is a live database, and get the node/queue to send the request to
isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName)
if err != nil {
Expand Down Expand Up @@ -1108,12 +1148,15 @@ func tablesHandler(w http.ResponseWriter, r *http.Request) {
// * "dbname" is the name of the database
func tagsHandler(w http.ResponseWriter, r *http.Request) {
// Do auth check, grab request info
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
if err != nil {
jsonErr(w, err.Error(), httpStatus)
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "tags", r.Header.Get("User-Agent"))

// If the database is a live database, we return an error message
isLive, _, err := com.CheckDBLive(dbOwner, dbName)
if err != nil {
Expand Down Expand Up @@ -1263,6 +1306,9 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, loggedInUser, dbName, "upload", r.Header.Get("User-Agent"))

// Check if the database exists already
exists, err := com.CheckDBExists(loggedInUser, dbName)
if err != nil {
Expand Down Expand Up @@ -1366,6 +1412,9 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, loggedInUser, dbName, "views", r.Header.Get("User-Agent"))

// Check if the database is a live database, and get the node/queue to send the request to
isLive, liveNode, err := com.CheckDBLive(dbOwner, dbName)
if err != nil {
Expand Down Expand Up @@ -1441,12 +1490,15 @@ func viewsHandler(w http.ResponseWriter, r *http.Request) {
// * "dbname" is the name of the database being queried
func webpageHandler(w http.ResponseWriter, r *http.Request) {
// Authenticate user and collect requested database details
_, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
loggedInUser, dbOwner, dbName, _, httpStatus, err := collectInfo(w, r)
if err != nil {
jsonErr(w, err.Error(), httpStatus)
return
}

// Record the api call in our backend database
com.ApiCallLog(loggedInUser, dbOwner, dbName, "views", r.Header.Get("User-Agent"))

// Return the database webUI URL to the user
var z com.WebpageResponseContainer
z.WebPage = "https://" + com.Conf.Web.ServerName + "/" + dbOwner + "/" + dbName
Expand Down
54 changes: 24 additions & 30 deletions api/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,6 @@ package main
// TODO: API functions that still need updating for Live databases
// * diff - already updated to just return an error for live databases. needs testing though

// FIXME: Update the documented Upload() function return values on the API doc page. Currently it talks about
// returning the commit ID for the upload. We'll probably return that field with a blank value for live
// databases though. TBD.

// FIXME: After the API and webui pieces are done, figure out how the DB4S end
// point and dio should be updated to use live databases too

Expand Down Expand Up @@ -113,29 +109,29 @@ func main() {
}

// Our pages
http.Handle("/", gz.GzipHandler(handleWrapper(rootHandler)))
http.Handle("/changelog", gz.GzipHandler(handleWrapper(changeLogHandler)))
http.Handle("/changelog.html", gz.GzipHandler(handleWrapper(changeLogHandler)))
http.Handle("/v1/branches", gz.GzipHandler(handleWrapper(branchesHandler)))
http.Handle("/v1/columns", gz.GzipHandler(handleWrapper(columnsHandler)))
http.Handle("/v1/commits", gz.GzipHandler(handleWrapper(commitsHandler)))
http.Handle("/v1/databases", gz.GzipHandler(handleWrapper(databasesHandler)))
http.Handle("/v1/delete", gz.GzipHandler(handleWrapper(deleteHandler)))
http.Handle("/v1/diff", gz.GzipHandler(handleWrapper(diffHandler)))
http.Handle("/v1/download", gz.GzipHandler(handleWrapper(downloadHandler)))
http.Handle("/v1/execute", gz.GzipHandler(handleWrapper(executeHandler)))
http.Handle("/v1/indexes", gz.GzipHandler(handleWrapper(indexesHandler)))
http.Handle("/v1/metadata", gz.GzipHandler(handleWrapper(metadataHandler)))
http.Handle("/v1/query", gz.GzipHandler(handleWrapper(queryHandler)))
http.Handle("/v1/releases", gz.GzipHandler(handleWrapper(releasesHandler)))
http.Handle("/v1/tables", gz.GzipHandler(handleWrapper(tablesHandler)))
http.Handle("/v1/tags", gz.GzipHandler(handleWrapper(tagsHandler)))
http.Handle("/v1/upload", gz.GzipHandler(handleWrapper(uploadHandler)))
http.Handle("/v1/views", gz.GzipHandler(handleWrapper(viewsHandler)))
http.Handle("/v1/webpage", gz.GzipHandler(handleWrapper(webpageHandler)))
http.Handle("/", gz.GzipHandler(corsWrapper(rootHandler)))
http.Handle("/changelog", gz.GzipHandler(corsWrapper(changeLogHandler)))
http.Handle("/changelog.html", gz.GzipHandler(corsWrapper(changeLogHandler)))
http.Handle("/v1/branches", gz.GzipHandler(corsWrapper(branchesHandler)))
http.Handle("/v1/columns", gz.GzipHandler(corsWrapper(columnsHandler)))
http.Handle("/v1/commits", gz.GzipHandler(corsWrapper(commitsHandler)))
http.Handle("/v1/databases", gz.GzipHandler(corsWrapper(databasesHandler)))
http.Handle("/v1/delete", gz.GzipHandler(corsWrapper(deleteHandler)))
http.Handle("/v1/diff", gz.GzipHandler(corsWrapper(diffHandler)))
http.Handle("/v1/download", gz.GzipHandler(corsWrapper(downloadHandler)))
http.Handle("/v1/execute", gz.GzipHandler(corsWrapper(executeHandler)))
http.Handle("/v1/indexes", gz.GzipHandler(corsWrapper(indexesHandler)))
http.Handle("/v1/metadata", gz.GzipHandler(corsWrapper(metadataHandler)))
http.Handle("/v1/query", gz.GzipHandler(corsWrapper(queryHandler)))
http.Handle("/v1/releases", gz.GzipHandler(corsWrapper(releasesHandler)))
http.Handle("/v1/tables", gz.GzipHandler(corsWrapper(tablesHandler)))
http.Handle("/v1/tags", gz.GzipHandler(corsWrapper(tagsHandler)))
http.Handle("/v1/upload", gz.GzipHandler(corsWrapper(uploadHandler)))
http.Handle("/v1/views", gz.GzipHandler(corsWrapper(viewsHandler)))
http.Handle("/v1/webpage", gz.GzipHandler(corsWrapper(webpageHandler)))

// favicon.ico
http.Handle("/favicon.ico", gz.GzipHandler(handleWrapper(func(w http.ResponseWriter, r *http.Request) {
http.Handle("/favicon.ico", gz.GzipHandler(corsWrapper(func(w http.ResponseWriter, r *http.Request) {
logReq(r, "-")
http.ServeFile(w, r, filepath.Join(com.Conf.Web.BaseDir, "webui", "favicon.ico"))
})))
Expand Down Expand Up @@ -329,9 +325,8 @@ func extractUserFromClientCert(w http.ResponseWriter, r *http.Request) (userAcc
return
}

// handleWrapper does nothing useful except interface between types
// TODO: Get rid of this, as it shouldn't be needed
func handleWrapper(fn http.HandlerFunc) http.HandlerFunc {
// corsWrapper sets a general allow for all our api calls
func corsWrapper(fn http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
// Enable CORS (https://enable-cors.org)
w.Header().Set("Access-Control-Allow-Origin", "*")
Expand Down Expand Up @@ -361,6 +356,5 @@ func jsonErr(w http.ResponseWriter, msg string, statusCode int) {
// logReq writes an entry for the incoming request to the request log
func logReq(r *http.Request, loggedInUser string) {
fmt.Fprintf(reqLog, "%v - %s [%s] \"%s %s %s\" \"-\" \"-\" \"%s\" \"%s\"\n", r.RemoteAddr,
loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto,
r.Referer(), r.Header.Get("User-Agent"))
loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto, r.Referer(), r.Header.Get("User-Agent"))
}
45 changes: 45 additions & 0 deletions common/postgresql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,51 @@ func AddUser(auth0ID, userName, password, email, displayName, avatarURL string)
return nil
}

// ApiCallLog records an API call operation. Database name is optional, as not all API calls operate on a
// database. If a database name is provided however, then the database owner name *must* also be provided
func ApiCallLog(loggedInUser, dbOwner, dbName, operation, callerSw string) {
var dbQuery string
var err error
var commandTag pgx.CommandTag
if dbName != "" {
dbQuery = `
WITH loggedIn AS (
SELECT user_id
FROM users
WHERE lower(user_name) = lower($1)
),
WITH owner AS (
SELECT user_id
FROM users
WHERE lower(user_name) = lower($2)
), d AS (
SELECT db.db_id
FROM sqlite_databases AS db, owner
WHERE db.user_id = owner.user_id
AND db.db_name = $3)
INSERT INTO api_call_log (caller_id, db_owner_id, db_id, api_operation, api_caller_sw)
VALUES ((SELECT user_id FROM loggedIn), (SELECT user_id FROM owner), (SELECT db_id FROM d), $4, $5)`
commandTag, err = pdb.Exec(dbQuery, loggedInUser, dbOwner, dbName, operation, callerSw)
} else {
dbQuery = `
WITH loggedIn AS (
SELECT user_id
FROM users
WHERE lower(user_name) = lower($1)
)
INSERT INTO api_call_log (caller_id, api_operation, api_caller_sw)
VALUES ((SELECT user_id FROM loggedIn), $2, $3)`
commandTag, err = pdb.Exec(dbQuery, loggedInUser, operation, callerSw)
}
if err != nil {
log.Printf("Adding api call log entry failed: %s", err)
return
}
if numRows := commandTag.RowsAffected(); numRows != 1 {
log.Printf("Wrong number of rows (%d) affected when adding api call entry for user '%s'", numRows, SanitiseLogString(loggedInUser))
}
}

// APIKeySave saves a new API key to the PostgreSQL database
func APIKeySave(key, loggedInUser string, dateCreated time.Time) error {
// Make sure the API key isn't already in the database
Expand Down
Loading

0 comments on commit 2364188

Please sign in to comment.