From 9c60872dcd927c497450de07a12edd454e801aca Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Thu, 24 Dec 2020 21:51:04 -0800 Subject: [PATCH 01/29] Simplify a line the team is provided in the authentication method; no need here --- frontend/src/api.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/src/api.js b/frontend/src/api.js index 6683eb84..9f8a5640 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -22,9 +22,8 @@ class Api { static newSubmission(submissionfile, callback){ // submissionfile.append('_method', 'PUT'); // get the url from the real api - $.post(`${URL}/api/${LEAGUE}/submission/`, { - team: Cookies.get('team_id') - }).done((data, status) => { + $.post(`${URL}/api/${LEAGUE}/submission/`) + .done((data, status) => { console.log("got URL") Cookies.set('submission_id', data['submission_id']); $.ajax({ From 62340a4417c78fce3b205b1d87257015877ea4ef Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Thu, 24 Dec 2020 21:54:57 -0800 Subject: [PATCH 02/29] Note that submissions should be enabled (game released being disabled prevents early submissions, anyways) --- backend/docs/SETUP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/docs/SETUP.md b/backend/docs/SETUP.md index 726ebe8d..9da0519a 100644 --- a/backend/docs/SETUP.md +++ b/backend/docs/SETUP.md @@ -52,7 +52,7 @@ Once the database is finished being created, connect to it with your Postgres ed Delete the contents of the following tables. (_Don't delete the tables themselves!_ To easily delete info, you can run a query, such as `DELETE FROM [table_name]`.) The tables are: `api_scrimmage`, `api_scrimmage_hidden`, `api_submission`, `api_team`, `api_team_users`, `api_teamsubmission`, `api_tournament`, `api_tournamentscrimmage`, `api_update`, `api_user`, `django_admin_log`. (You may have to delete them in a particular order. Particularly, if you get an error pertaining to a "foreign key constraint", you'll have to delete the table which uses it first. Deleting those tables is probably okay.) -Updating `api_league` is slightly different. Don't delete the entry; just edit it instead. Change `name` to something more suitable (eg `bh20`), change the `start_date` and `end_date` (they don't have to be exact, so feel free to use a longer range than the actual tournament. Set `active` to true. **Set `submissions_enabled` to false and `game_released` to false.** Finally `engine_version` needs to be changed as well; ask the infrastructure team what to change it to. +Updating `api_league` is slightly different. Don't delete the entry; just edit it instead. Change `name` to something more suitable (eg `bh20`), change the `start_date` and `end_date` (they don't have to be exact, so feel free to use a longer range than the actual tournament. **Set `active` to true. Set `submissions_enabled` to true. Set `game_released` to false.** Finally `engine_version` needs to be changed as well; ask the infrastructure team what to change it to. Next, we need to register a superuser account (for use by the infra). Run the battlecode website, and simply follow the normal account registration process. Take note of the password! Also, have this superuser create and join a team (this is necessary for some permissions). From 080fe9154063bd96fa6115e474aec55d5343b876 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Thu, 24 Dec 2020 21:55:24 -0800 Subject: [PATCH 03/29] TODO write up cors things --- backend/docs/SETUP.md | 2 ++ cors.json | 1 + 2 files changed, 3 insertions(+) create mode 100644 cors.json diff --git a/backend/docs/SETUP.md b/backend/docs/SETUP.md index 9da0519a..09de0012 100644 --- a/backend/docs/SETUP.md +++ b/backend/docs/SETUP.md @@ -110,3 +110,5 @@ Finally, click update! Delete old instance groups: go to "Compute Engine" -> "Instance groups", check any old instance groups that are no longer in use, and click "delete". Delete old instance template: go to "Compute Engine" -> "Instance templates", check any old templates that are no longer in use, and click "delete". Delete old, unused backend services and buckets, if you're up to it, instructions in previous section. But this can be a pain and is certainly not necessary. + + diff --git a/cors.json b/cors.json new file mode 100644 index 00000000..e4ce8b35 --- /dev/null +++ b/cors.json @@ -0,0 +1 @@ +[{"method": ["GET", "PUT"], "origin": ["http://localhost:3000", "http://2021.battlecode.org", "https://2021.battlecode.org"], "responseHeader": ["authorization"]}] From 79f6e00059f82c19d88088f96a8f54c4a6f192a7 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Thu, 24 Dec 2020 22:16:11 -0800 Subject: [PATCH 04/29] Listing more TODOs --- backend/docs/SETUP.md | 2 ++ backend/settings.py | 1 + 2 files changed, 3 insertions(+) diff --git a/backend/docs/SETUP.md b/backend/docs/SETUP.md index 09de0012..153068c1 100644 --- a/backend/docs/SETUP.md +++ b/backend/docs/SETUP.md @@ -112,3 +112,5 @@ Delete old instance template: go to "Compute Engine" -> "Instance templates", ch Delete old, unused backend services and buckets, if you're up to it, instructions in previous section. But this can be a pain and is certainly not necessary. + diff --git a/backend/settings.py b/backend/settings.py index 4a071fce..da1be4e9 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -272,4 +272,5 @@ class COMPILE_STATUS: # google cloud +# TODO the env name shouldn't have json at the end. Remember to change deployment too!!!! GOOGLE_APPLICATION_CREDENTIALS = os.getenv('GOOGLE_APPLICATION_CREDENTIALS_JSON') From a13ccdbbb946bb2439bea5a84521918c927fd879 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 11:55:46 -0800 Subject: [PATCH 05/29] Add bucket CORS setup notes --- backend/docs/SETUP.md | 10 ++++++++-- cors.json => backend/docs/cors.json | 0 2 files changed, 8 insertions(+), 2 deletions(-) rename cors.json => backend/docs/cors.json (100%) diff --git a/backend/docs/SETUP.md b/backend/docs/SETUP.md index 153068c1..0656105c 100644 --- a/backend/docs/SETUP.md +++ b/backend/docs/SETUP.md @@ -62,7 +62,14 @@ Then stop the old database (on its main page, press "stop"). ## Deployment Setup -Deployment is done through the Google Cloud Platform. You'll need access to the Google Cloud project. (If you don't have access already, ask a dev to add you.) With that, you can start here: +Deployment is done through the Google Cloud Platform. You'll need access to the Google Cloud project. (If you don't have access already, ask a dev to add you.) It's also helpful to install gsutil, a command line application for managing GCP. Link here: https://cloud.google.com/storage/docs/gsutil. + +With that, you can start here -- + +### Storage Buckets +Go to "Storage" on GCP console. A bucket for submissions should have been created (if not, instructions are in the infrastructure readme.) +Set up the CORS policy, which allows us to upload to the bucket on external websites. Find `docs/cors,json`; in there, update the domain URLs listed. Then, run `gsutil cors set path/to/cors.json gs://bc21-submissions` (updating the bucket name) to whatever it is this year. +More info is here: https://cloud.google.com/storage/docs/configuring-cors#gsutil ### Cloud Build Triggers Go to "Cloud Build" triggers on GCP console, here: https://console.cloud.google.com/cloud-build/triggers?project=battlecode18 @@ -111,6 +118,5 @@ Delete old instance groups: go to "Compute Engine" -> "Instance groups", check a Delete old instance template: go to "Compute Engine" -> "Instance templates", check any old templates that are no longer in use, and click "delete". Delete old, unused backend services and buckets, if you're up to it, instructions in previous section. But this can be a pain and is certainly not necessary. - diff --git a/cors.json b/backend/docs/cors.json similarity index 100% rename from cors.json rename to backend/docs/cors.json From 3e4562dc6ed2e241c531c018b0b7b790909b68ae Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 12:15:56 -0800 Subject: [PATCH 06/29] Notes about CORS, Google App creds --- backend/docs/SETUP.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/docs/SETUP.md b/backend/docs/SETUP.md index 0656105c..0b94e6ee 100644 --- a/backend/docs/SETUP.md +++ b/backend/docs/SETUP.md @@ -79,11 +79,15 @@ Change Dockerfile directory to `/backend`, and image name to `gcr.io/battlecode1 With this step done: on pushes to master, Google Cloud will create a Docker container with our latest code. Push a commit to master, to test that the trigger works! Under "Cloud Builds" -> "History" you can see the build in progress. +### Google Application Credentials +Infrastructure should have made a service account. Get the service account json file from an infra dev. (If they haven't done so yet, you can come back to this section later. Make sure to!) +Set the contents of this file into dev_settings_sensitive.py, as GOOGLE_APPLICATION_CREDENTIALS. Formatting is a little weird here -- you'll have to wrap the contents of the json file in `r'''` at the beginning, and `'''` at the end. See another version of the file for an example. + ### Instance Template From Google Cloud console, "Compute Engine" -> “Instance Templates”. Click on an old backend template, and then click on “Create similar”. Change the name to something descriptive enough and conventional. ("bc21-backend-template", for example, works well. Also I’ve found that including the current date and time in the name can help keep things straight.) For machine type, we've found the `n1-standard-n1` to be cheap and work well, especially providing enough memory. -Check the checkbox of "Deploy a container image to this VM instance", and change the container image to the image name you've just written in the cloud build trigger. -Then, click "Advanced container options" to see a place to set environment variables. In the repo's `backend/settings.py`, you can look at the `os.getenv` calls to see which environment variables are needed. Set these here, to the same values that have been used in local testing / in `dev_settings_sensitive.py`. (Other than `DB_HOST`, these probably don't need changing.) Note that these are un-editable; if you ever change environment variables, you'll have to make a new instance template. +Check the checkbox of "Deploy a container image to this VM instance", and change the container image to the image name you've just written in the cloud build trigger. +Then, click "Advanced container options" to see a place to set environment variables. Find the variables set in `dev_settings_sensitive.py`, and set all of those keys/values here, too. (Here, these values should not be enclosed in quotes) Note that these are un-editable; if you ever change environment variables, you'll have to make a new instance template. (For now, keep the boot disk the same; it may be good to change it to a later version down the road. Be sure to test that the VMs still work, though.) @@ -114,9 +118,7 @@ Finally, click update! (Note: sometimes, after you try to update changes, they may not go through. This may be due to creating too many backend instances/buckets; we can only have so many up at any given time. You'll see notifications and any errors in the top right corner of the Google Console; you can check if this is the problem. If so, deleting old backend services/buckets is surprisingly hard. You need to first delete any uses of them in the host and path rules, then delete their uses in the "backend services" / "backend buckets" lists on the edit page's backend configuration section; don't forget to save. Then you need to _actually_ delete them, by using the gcloud command line interface. Instructions [here](https://cloud.google.com/sdk/gcloud/reference/compute/backend-services/delete) and [here](https://cloud.google.com/sdk/gcloud/reference/compute/backend-buckets/delete).) ### Some last steps +Make sure the CORS policy and Google Application credentials are all set up, as described earlier. In particular make sure that the Google Application credentials have been set up as an environment variable in the instance template, or create a new instance template with this set. Delete old instance groups: go to "Compute Engine" -> "Instance groups", check any old instance groups that are no longer in use, and click "delete". Delete old instance template: go to "Compute Engine" -> "Instance templates", check any old templates that are no longer in use, and click "delete". Delete old, unused backend services and buckets, if you're up to it, instructions in previous section. But this can be a pain and is certainly not necessary. - - From 0fd11d56180f6bdaae99206766ad07185d7088e4 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 12:28:30 -0800 Subject: [PATCH 07/29] Improve setting google app creds env in deployment --- backend/docs/SETUP.md | 2 +- backend/settings.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/docs/SETUP.md b/backend/docs/SETUP.md index 0b94e6ee..9f2b2e88 100644 --- a/backend/docs/SETUP.md +++ b/backend/docs/SETUP.md @@ -87,7 +87,7 @@ Set the contents of this file into dev_settings_sensitive.py, as GOOGLE_APPLICAT From Google Cloud console, "Compute Engine" -> “Instance Templates”. Click on an old backend template, and then click on “Create similar”. Change the name to something descriptive enough and conventional. ("bc21-backend-template", for example, works well. Also I’ve found that including the current date and time in the name can help keep things straight.) For machine type, we've found the `n1-standard-n1` to be cheap and work well, especially providing enough memory. Check the checkbox of "Deploy a container image to this VM instance", and change the container image to the image name you've just written in the cloud build trigger. -Then, click "Advanced container options" to see a place to set environment variables. Find the variables set in `dev_settings_sensitive.py`, and set all of those keys/values here, too. (Here, these values should not be enclosed in quotes) Note that these are un-editable; if you ever change environment variables, you'll have to make a new instance template. +Then, click "Advanced container options" to see a place to set environment variables. Find the variables set in `dev_settings_sensitive.py`, and set all of those keys/values here, too. (Here, these values should not be enclosed in quotes.) Note that these are un-editable; if you ever change environment variables, you'll have to make a new instance template. ("Create Similar" on the instance template's page is helpful here.) (For now, keep the boot disk the same; it may be good to change it to a later version down the road. Be sure to test that the VMs still work, though.) diff --git a/backend/settings.py b/backend/settings.py index da1be4e9..fb2e634e 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -272,5 +272,4 @@ class COMPILE_STATUS: # google cloud -# TODO the env name shouldn't have json at the end. Remember to change deployment too!!!! -GOOGLE_APPLICATION_CREDENTIALS = os.getenv('GOOGLE_APPLICATION_CREDENTIALS_JSON') +GOOGLE_APPLICATION_CREDENTIALS = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') From 9fc6f52c02c9d17ff7fc5624b259b78b69fdd4bd Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 20:14:10 -0800 Subject: [PATCH 08/29] Set upload origin for proper cors --- backend/api/views.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/api/views.py b/backend/api/views.py index a9768667..ce421494 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -140,7 +140,14 @@ def signed_upload_url(file_path, bucket): """ blob = GCloudUploadDownload.get_blob(file_path, bucket) - return blob.create_resumable_upload_session() + # Origin is necessary to prevent CORS errors later: + # https://stackoverflow.com/questions/25688608/xmlhttprequest-cors-to-google-cloud-storage-only-working-in-preflight-request + # https://stackoverflow.com/questions/46971451/cors-request-made-despite-error-in-console + # https://googleapis.dev/python/storage/latest/blobs.html + # TODO this should be dynamic. + # in fact, "this website" should become a config setting; + # also, in the setup readme, should be a note about changing this setting + return blob.create_resumable_upload_session(origin='http://localhost:3000') @staticmethod def signed_download_url(file_path, bucket): From edd752f82fb4fb8bd8a043807409898736305deb Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 20:19:15 -0800 Subject: [PATCH 09/29] Format json --- backend/docs/cors.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/backend/docs/cors.json b/backend/docs/cors.json index e4ce8b35..f29d8e22 100644 --- a/backend/docs/cors.json +++ b/backend/docs/cors.json @@ -1 +1,16 @@ -[{"method": ["GET", "PUT"], "origin": ["http://localhost:3000", "http://2021.battlecode.org", "https://2021.battlecode.org"], "responseHeader": ["authorization"]}] +[ + { + "method": [ + "GET", + "PUT" + ], + "origin": [ + "http://localhost:3000", + "http://2021.battlecode.org", + "https://2021.battlecode.org" + ], + "responseHeader": [ + "authorization" + ] + } +] \ No newline at end of file From 084f146a3e514ea6f66f5ad3af34fa35b713d715 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 20:26:06 -0800 Subject: [PATCH 10/29] Extra header tha t might be necessary (sneaks into access-control-request-headers I think) --- backend/docs/cors.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/docs/cors.json b/backend/docs/cors.json index f29d8e22..c66e64cc 100644 --- a/backend/docs/cors.json +++ b/backend/docs/cors.json @@ -10,7 +10,8 @@ "https://2021.battlecode.org" ], "responseHeader": [ - "authorization" + "authorization", + "content-type" ] } ] \ No newline at end of file From f03b249b336fcdc005fa1bfdc84363e5003cd225 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 21:22:05 -0800 Subject: [PATCH 11/29] Move compilation update callback to on success (since the upload doesn't throw an error now!) --- frontend/src/api.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/frontend/src/api.js b/frontend/src/api.js index 9f8a5640..d8706f63 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -35,17 +35,6 @@ class Api { }) .done((data, status) => { console.log(data, status) - }) - // Even when upload succeeds, an error is thrown... - // We make the dangerous assumption that the upload succeeded, - // ie that the submission exists in a bucket - // TODO this is a dangerous assumption, find a better solution - // (maybe revolving around the upload working error-free, - // and hooking callbacks to done rather than fail) - // TODO it's possible that the fail callback occurs - // before the upload finishes - .fail((xhr, status, error) => { - // console.log(data); $.post(`${URL}/api/${LEAGUE}/submission/` +Cookies.get('submission_id') + `/compilation_update/`, { team: Cookies.get('team_id') }).done((data, status) => { From 463919f649e2977a01b89ff2cd3dfc7cc321eb0b Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 21:22:31 -0800 Subject: [PATCH 12/29] Better error logging/formatting --- frontend/src/api.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/frontend/src/api.js b/frontend/src/api.js index d8706f63..d78ad02f 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -37,16 +37,23 @@ class Api { console.log(data, status) $.post(`${URL}/api/${LEAGUE}/submission/` +Cookies.get('submission_id') + `/compilation_update/`, { team: Cookies.get('team_id') - }).done((data, status) => { + }) + .done((data, status) => { console.log("Definitely done!") // console.log(data, status) Cookies.set('submitting', 0) // TODO make this display done on screen }) + .fail((xhr, status, error) => { + console.log("Error in compilation update callback: ", xhr, status, error) + }) }) - }).fail((xhr, status, error) => { - console.log("Error in post:", error) - + .fail((xhr, status, error) => { + console.log("Error in put request of file to bucket: ", xhr, status, error) + }) + }) + .fail((xhr, status, error) => { + console.log("Error in post request for upload: ", xhr, status, error) }); } From 428fa512647f1a3eac82ce0259a773358fda1e9d Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 21:33:34 -0800 Subject: [PATCH 13/29] Add a bunch of TODOs for in this branch --- frontend/src/api.js | 6 ++++++ frontend/src/views/submissions.js | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/frontend/src/api.js b/frontend/src/api.js index d78ad02f..33cb71f7 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -18,6 +18,12 @@ class Api { //----SUBMISSIONS---- + // TODO clean up a lot of old comments, print statements + // TODO provide more explanatory comments + // TODO there's a better wayy to work with 'submitting' in cookies + // TODO 'submitting' could probably use a better name + // TODO review code in the submissions js + //uploads a new submission to the google cloud bucket static newSubmission(submissionfile, callback){ // submissionfile.append('_method', 'PUT'); diff --git a/frontend/src/views/submissions.js b/frontend/src/views/submissions.js index 229b28e4..a2b34ffe 100755 --- a/frontend/src/views/submissions.js +++ b/frontend/src/views/submissions.js @@ -46,6 +46,11 @@ class Submissions extends Component { // makes an api call to upload the selected file + // TODO clean this method up + // TODO add explanation + // TODO submission table should be what exactly? + // Latest submission in progress, and last 3 good submissions? (and then make this clear in frontend) -- think I'm leaning towards this one + // Last 3 submissions, period? (this might need revisions in backend) uploadData = () => { // let status_str = "Submitting..." Cookies.set('submitting', 1) From 7aac8ace1c2994337100e55bc8cdfa7ec72da578 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Sun, 27 Dec 2020 21:37:52 -0800 Subject: [PATCH 14/29] moar TODOs --- frontend/src/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/api.js b/frontend/src/api.js index 33cb71f7..1e865848 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -23,6 +23,7 @@ class Api { // TODO there's a better wayy to work with 'submitting' in cookies // TODO 'submitting' could probably use a better name // TODO review code in the submissions js + // TODO errors in these callbacks should also display messages in frontend //uploads a new submission to the google cloud bucket static newSubmission(submissionfile, callback){ From d9ed64e51cce7a4bc3a2ced54530cd7b3c1986d9 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 09:16:26 -0800 Subject: [PATCH 15/29] Even more TODOs --- backend/api/views.py | 2 ++ frontend/src/api.js | 3 +++ 2 files changed, 5 insertions(+) diff --git a/backend/api/views.py b/backend/api/views.py index ce421494..00354b5e 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -684,6 +684,7 @@ def create(self, request, team, league_id): upload_url = GCloudUploadDownload.signed_upload_url(SUBMISSION_FILENAME(serializer.data['id']), GCLOUD_SUB_BUCKET) + # TODO everything that this comment talks about # The submission process is problematic: if the IDs are recorded, before the code is actually uploaded, then code that fails to upload will have dead IDs associated with it, and the team will be sad # Also, if user navigates away before the upload_url is returned, # then no code makes it into the bucket @@ -827,6 +828,7 @@ def team_compilation_id(self, request, team, league_id, pk=None): return Response({'compilation_id': comp_id}, status.HTTP_200_OK) else: # this is bad, replace with something thats actually None + # ^ TODO should address this return Response({'compilation_id': -1}, status.HTTP_200_OK) diff --git a/frontend/src/api.js b/frontend/src/api.js index 1e865848..0c3da073 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -42,6 +42,9 @@ class Api { }) .done((data, status) => { console.log(data, status) + // TODO instead of setting compilation_update, this should (I think) shove something onto to compile pub/sub, + // through a backend endpoint. + // (probably can see bc20 code for help, although the order of when things occur may need changes from original, see #101 and (less relevantly) #42 for some discussion. #10 may also be relevant, too.) $.post(`${URL}/api/${LEAGUE}/submission/` +Cookies.get('submission_id') + `/compilation_update/`, { team: Cookies.get('team_id') }) From 9f8f2723f73a8560f57bb3f35f8bdd686f4aa646 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 12:24:49 -0800 Subject: [PATCH 16/29] Add newline at end of file --- backend/docs/cors.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/docs/cors.json b/backend/docs/cors.json index c66e64cc..de4be662 100644 --- a/backend/docs/cors.json +++ b/backend/docs/cors.json @@ -14,4 +14,4 @@ "content-type" ] } -] \ No newline at end of file +] From 61ea08802763aa3467869f3cb45940faa189c772 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 12:35:51 -0800 Subject: [PATCH 17/29] THIS_URL setting improve, use in cors origin --- backend/api/views.py | 5 +---- backend/dev_settings.py | 1 + backend/docs/SETUP.md | 4 ++++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/api/views.py b/backend/api/views.py index 00354b5e..c0091bf1 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -144,10 +144,7 @@ def signed_upload_url(file_path, bucket): # https://stackoverflow.com/questions/25688608/xmlhttprequest-cors-to-google-cloud-storage-only-working-in-preflight-request # https://stackoverflow.com/questions/46971451/cors-request-made-despite-error-in-console # https://googleapis.dev/python/storage/latest/blobs.html - # TODO this should be dynamic. - # in fact, "this website" should become a config setting; - # also, in the setup readme, should be a note about changing this setting - return blob.create_resumable_upload_session(origin='http://localhost:3000') + return blob.create_resumable_upload_session(origin=settings.THIS_URL) @staticmethod def signed_download_url(file_path, bucket): diff --git a/backend/dev_settings.py b/backend/dev_settings.py index 26a32776..197a022f 100644 --- a/backend/dev_settings.py +++ b/backend/dev_settings.py @@ -40,6 +40,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True +THIS_URL = 'http://localhost:3000' # Application definition INSTALLED_APPS += ['debug_toolbar'] diff --git a/backend/docs/SETUP.md b/backend/docs/SETUP.md index 9f2b2e88..2a250df9 100644 --- a/backend/docs/SETUP.md +++ b/backend/docs/SETUP.md @@ -66,6 +66,10 @@ Deployment is done through the Google Cloud Platform. You'll need access to the With that, you can start here -- +### Configuring Settings + +After registering a domain name for the competition, set `THIS_URL` (in `settings.py`) to that domain. + ### Storage Buckets Go to "Storage" on GCP console. A bucket for submissions should have been created (if not, instructions are in the infrastructure readme.) Set up the CORS policy, which allows us to upload to the bucket on external websites. Find `docs/cors,json`; in there, update the domain URLs listed. Then, run `gsutil cors set path/to/cors.json gs://bc21-submissions` (updating the bucket name) to whatever it is this year. From c4cee4929cb524d6d77042c0c5c5079acf7e2f72 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 12:45:14 -0800 Subject: [PATCH 18/29] Add call to compile pub/sub --- backend/api/views.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/api/views.py b/backend/api/views.py index c0091bf1..a19951fb 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -691,6 +691,14 @@ def create(self, request, team, league_id): # call a function in the backend that adjusts sub IDs # TODO somehow fix this problem + # call to compile server + print('attempting call to compile server') + print('id:', serializer.data['id']) + data = str(serializer.data['id']) + data_bytestring = data.encode('utf-8') + print(type(data_bytestring)) + pub(GCLOUD_PROJECT, GCLOUD_SUB_COMPILE_NAME, data_bytestring) + return Response({'upload_url': upload_url, 'submission_id': submission.id}, status.HTTP_201_CREATED) From 80d26707e1240e182d4b827b63d949563a2cfe54 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 14:07:53 -0800 Subject: [PATCH 19/29] In-progress Endpoint to call compilation pubsub --- backend/api/views.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/backend/api/views.py b/backend/api/views.py index a19951fb..61f2964d 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -721,6 +721,27 @@ def retrieve_file(self, request, team, league_id, pk=None): return Response({'download_url': download_url}, status.HTTP_200_OK) + @action(methods=['patch', 'post'], detail=True) + def compilation_pubsub_call(self, request, team, league_id, pk=None): + submission = self.get_queryset().get(pk=pk) + if team != submission.team: + return Response({'message': 'Not authenticated on the right team'}, status.HTTP_401_UNAUTHORIZED) + + # TODO checks for old status, before setting + # (eg a submission w status success shouldn't be tossed on the pubsub again.) + # (might make sense for any other status to be tossed on again, though) + + # TODO save to a status that indicates confirmation of submission being in a bucket + submission.compilation_status = 0 + submission.save() + + # TODO call the pubsub here + + # TODO if message successfully added to pubsub, + # change to a status that indicates being queued + + return Response({'message': 'Status updated'}, status.HTTP_200_OK) + @action(methods=['patch', 'post'], detail=True) def compilation_update(self, request, team, league_id, pk=None): is_admin = User.objects.all().get(username=request.user).is_superuser From bdb5ff64b1e41ee0bd9fcd75a909e0904d73f1b3 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 14:14:00 -0800 Subject: [PATCH 20/29] New compile statuses --- backend/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/settings.py b/backend/settings.py index fb2e634e..53e65e16 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -104,6 +104,8 @@ class COMPILE_STATUS: SUCCESS = 1 FAIL = 2 ERROR = 3 + UPLOADED = 4 + QUEUED = 5 # Application definition From e606db937e891b9d4feea7967a66976e421e1dd6 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 14:20:19 -0800 Subject: [PATCH 21/29] Implement use of new status codes --- backend/api/views.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/api/views.py b/backend/api/views.py index 61f2964d..ad364f3f 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -727,18 +727,20 @@ def compilation_pubsub_call(self, request, team, league_id, pk=None): if team != submission.team: return Response({'message': 'Not authenticated on the right team'}, status.HTTP_401_UNAUTHORIZED) - # TODO checks for old status, before setting - # (eg a submission w status success shouldn't be tossed on the pubsub again.) - # (might make sense for any other status to be tossed on again, though) + # If a compilation has already succeeded, keep as so; no need to re-do. + # (Might make sense to re-do for other submissions, however.) + if submission.compilation_status == settings.COMPILE_STATUS.SUCCESS: + return Response({'message': 'Success response already received for this submission'}, status.HTTP_400_BAD_REQUEST) - # TODO save to a status that indicates confirmation of submission being in a bucket - submission.compilation_status = 0 + # indicate submission being in a bucket + submission.compilation_status = settings.COMPILE_STATUS.UPLOADED submission.save() # TODO call the pubsub here - # TODO if message successfully added to pubsub, - # change to a status that indicates being queued + # indicate submission being queued + submission.compilation_status = settings.COMPILE_STATUS.QUEUED + submission.save() return Response({'message': 'Status updated'}, status.HTTP_200_OK) From 2fa0fa955f449bdd9cf224d993e2ec6f7e2b82fc Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 14:22:25 -0800 Subject: [PATCH 22/29] Put the pub-sub call where it should be --- backend/api/views.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/backend/api/views.py b/backend/api/views.py index ad364f3f..0c3bb88a 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -691,14 +691,6 @@ def create(self, request, team, league_id): # call a function in the backend that adjusts sub IDs # TODO somehow fix this problem - # call to compile server - print('attempting call to compile server') - print('id:', serializer.data['id']) - data = str(serializer.data['id']) - data_bytestring = data.encode('utf-8') - print(type(data_bytestring)) - pub(GCLOUD_PROJECT, GCLOUD_SUB_COMPILE_NAME, data_bytestring) - return Response({'upload_url': upload_url, 'submission_id': submission.id}, status.HTTP_201_CREATED) @@ -736,7 +728,13 @@ def compilation_pubsub_call(self, request, team, league_id, pk=None): submission.compilation_status = settings.COMPILE_STATUS.UPLOADED submission.save() - # TODO call the pubsub here + # call to compile server + print('attempting call to compile server') + print('id:', serializer.data['id']) + data = str(serializer.data['id']) + data_bytestring = data.encode('utf-8') + print(type(data_bytestring)) + pub(GCLOUD_PROJECT, GCLOUD_SUB_COMPILE_NAME, data_bytestring) # indicate submission being queued submission.compilation_status = settings.COMPILE_STATUS.QUEUED From 0304c5f672e74760d7a132fc19339efbb406ca89 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 14:24:02 -0800 Subject: [PATCH 23/29] Serializer, data needed --- backend/api/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/api/views.py b/backend/api/views.py index 0c3bb88a..800f4e43 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -728,6 +728,10 @@ def compilation_pubsub_call(self, request, team, league_id, pk=None): submission.compilation_status = settings.COMPILE_STATUS.UPLOADED submission.save() + data = { + 'team': team.id + } + serializer = self.get_serializer(data=data) # call to compile server print('attempting call to compile server') print('id:', serializer.data['id']) From a2e05c061c32c83230383ce32ad462b24eacbf3f Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 14:35:18 -0800 Subject: [PATCH 24/29] Properly get submission id --- backend/api/views.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/api/views.py b/backend/api/views.py index 800f4e43..efee7694 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -728,14 +728,11 @@ def compilation_pubsub_call(self, request, team, league_id, pk=None): submission.compilation_status = settings.COMPILE_STATUS.UPLOADED submission.save() - data = { - 'team': team.id - } - serializer = self.get_serializer(data=data) + id = submission.id # call to compile server print('attempting call to compile server') - print('id:', serializer.data['id']) - data = str(serializer.data['id']) + print('id:', id) + data = str(id) data_bytestring = data.encode('utf-8') print(type(data_bytestring)) pub(GCLOUD_PROJECT, GCLOUD_SUB_COMPILE_NAME, data_bytestring) From 7dd939347cc3184aadb038fde0f76f8184974272 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 14:49:14 -0800 Subject: [PATCH 25/29] Frontend pokes the new endpoint --- frontend/src/api.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/src/api.js b/frontend/src/api.js index 0c3da073..4e561377 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -45,9 +45,7 @@ class Api { // TODO instead of setting compilation_update, this should (I think) shove something onto to compile pub/sub, // through a backend endpoint. // (probably can see bc20 code for help, although the order of when things occur may need changes from original, see #101 and (less relevantly) #42 for some discussion. #10 may also be relevant, too.) - $.post(`${URL}/api/${LEAGUE}/submission/` +Cookies.get('submission_id') + `/compilation_update/`, { - team: Cookies.get('team_id') - }) + $.post(`${URL}/api/${LEAGUE}/submission/` +Cookies.get('submission_id') + `/compilation_pubsub_call/`) .done((data, status) => { console.log("Definitely done!") // console.log(data, status) From fd42b259ee6b5269143e49312cb53d3ce67f6f85 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 14:58:09 -0800 Subject: [PATCH 26/29] Explanatory notes --- backend/api/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/api/views.py b/backend/api/views.py index efee7694..f6ded5df 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -663,6 +663,7 @@ def create(self, request, team, league_id): if not serializer.is_valid(): return Response(serializer.errors, status.HTTP_400_BAD_REQUEST) + # Note that IDs are needed to generate the link. serializer.save() #save it once, link will be undefined since we don't have any way to know id serializer.save() #save again, link automatically set @@ -715,6 +716,8 @@ def retrieve_file(self, request, team, league_id, pk=None): @action(methods=['patch', 'post'], detail=True) def compilation_pubsub_call(self, request, team, league_id, pk=None): + # It is better if compile server gets requests for compiling submissions that are actually in buckets. + # So, only after an upload is done, the frontend calls this endpoint to give the compile server a request. submission = self.get_queryset().get(pk=pk) if team != submission.team: return Response({'message': 'Not authenticated on the right team'}, status.HTTP_401_UNAUTHORIZED) From 6b835e1617d39e57424c988f6564e38fd6cc6263 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 15:16:16 -0800 Subject: [PATCH 27/29] remove some completed TODOs --- frontend/src/api.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frontend/src/api.js b/frontend/src/api.js index 4e561377..b5719ddd 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -42,15 +42,11 @@ class Api { }) .done((data, status) => { console.log(data, status) - // TODO instead of setting compilation_update, this should (I think) shove something onto to compile pub/sub, - // through a backend endpoint. - // (probably can see bc20 code for help, although the order of when things occur may need changes from original, see #101 and (less relevantly) #42 for some discussion. #10 may also be relevant, too.) $.post(`${URL}/api/${LEAGUE}/submission/` +Cookies.get('submission_id') + `/compilation_pubsub_call/`) .done((data, status) => { console.log("Definitely done!") // console.log(data, status) Cookies.set('submitting', 0) - // TODO make this display done on screen }) .fail((xhr, status, error) => { console.log("Error in compilation update callback: ", xhr, status, error) From e0895ee7aa72451e4785f7c774595801e48a9193 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 15:49:58 -0800 Subject: [PATCH 28/29] More explanatory notes --- backend/api/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/api/views.py b/backend/api/views.py index f6ded5df..8e6f3f80 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -688,6 +688,7 @@ def create(self, request, team, league_id): # then no code makes it into the bucket # This is fixed(?) by uploading in the backend, # or by uploading the file and then pressing another button to officialy submit + # (note: ended up doing this^, except instead of a button press, the submission process is automatically continued along) # The best way for now would be to have the upload, when done, # call a function in the backend that adjusts sub IDs # TODO somehow fix this problem From 1b2a08bdce03f629db16f7638c5db3709d749ce7 Mon Sep 17 00:00:00 2001 From: Nathaniel Kim Date: Mon, 28 Dec 2020 15:51:49 -0800 Subject: [PATCH 29/29] Updates to TODOs (revisions, better places for them) --- backend/api/views.py | 11 ----------- frontend/src/views/submissions.js | 4 ++++ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/backend/api/views.py b/backend/api/views.py index 8e6f3f80..8bb0cdc3 100644 --- a/backend/api/views.py +++ b/backend/api/views.py @@ -682,17 +682,6 @@ def create(self, request, team, league_id): upload_url = GCloudUploadDownload.signed_upload_url(SUBMISSION_FILENAME(serializer.data['id']), GCLOUD_SUB_BUCKET) - # TODO everything that this comment talks about - # The submission process is problematic: if the IDs are recorded, before the code is actually uploaded, then code that fails to upload will have dead IDs associated with it, and the team will be sad - # Also, if user navigates away before the upload_url is returned, - # then no code makes it into the bucket - # This is fixed(?) by uploading in the backend, - # or by uploading the file and then pressing another button to officialy submit - # (note: ended up doing this^, except instead of a button press, the submission process is automatically continued along) - # The best way for now would be to have the upload, when done, - # call a function in the backend that adjusts sub IDs - # TODO somehow fix this problem - return Response({'upload_url': upload_url, 'submission_id': submission.id}, status.HTTP_201_CREATED) diff --git a/frontend/src/views/submissions.js b/frontend/src/views/submissions.js index a2b34ffe..ea21119e 100755 --- a/frontend/src/views/submissions.js +++ b/frontend/src/views/submissions.js @@ -51,6 +51,10 @@ class Submissions extends Component { // TODO submission table should be what exactly? // Latest submission in progress, and last 3 good submissions? (and then make this clear in frontend) -- think I'm leaning towards this one // Last 3 submissions, period? (this might need revisions in backend) + // TODO update how we display the most recent submission (including its status.) + // Also now that we have new statuses, we need to figue out what we should display in the frontend for each of them. + // (eg if user navigates away before the upload link is returned / before the upload finishes, or if submission fails to get queued/compiled, + // what should the user do? what should we tell them?) uploadData = () => { // let status_str = "Submitting..." Cookies.set('submitting', 1)