Experiment with extra check for an uploaded file

upload-server/cmd/cli/info.go

@@ -6,12 +6,15 @@ import (
  "errors"
  "fmt"
  "net/http"
+ "time"
 
  "github.com/cdcgov/data-exchange-upload/upload-server/internal/appconfig"
  "github.com/cdcgov/data-exchange-upload/upload-server/internal/storeaz"
  "github.com/cdcgov/data-exchange-upload/upload-server/pkg/azureinspector"
  "github.com/cdcgov/data-exchange-upload/upload-server/pkg/fileinspector"
  "github.com/cdcgov/data-exchange-upload/upload-server/pkg/info"
+ "github.com/tus/tusd/v2/pkg/handler"
+ "github.com/tus/tusd/v2/pkg/hooks"
 )
 
 type UploadInspecter interface {
@@ -46,6 +49,49 @@ func (ih *InfoHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
  enc.Encode(response)
 }
 
+func (ih *InfoHandler) Hook(event handler.HookEvent, resp hooks.HookResponse) (hooks.HookResponse, error) {
+ id := event.Upload.ID
+ ctx := event.Context
+
+ var fileInfo map[string]any
+ for i := 0; i < 5; i++ {
+ var err error
+ fileInfo, err = ih.inspecter.InspectInfoFile(ctx, id)
+ if err != nil {
+ logger.Info("Failed to validate manifest file after upload", "id", id, "error", err, "manifest", fileInfo)
+ }
+ if fileInfo != nil {
+ break
+ }
+ time.Sleep(1 * time.Second)
+ }
+ var uploadedFileInfo map[string]any
+ for i := 0; i < 5; i++ {
+ var err error
+ uploadedFileInfo, err = ih.inspecter.InspectUploadedFile(ctx, id)
+ if err != nil {
+ logger.Info("Failed to validate upload file", "id", id, "error", err, "manifest", fileInfo, "upload", uploadedFileInfo)
+ }
+ if uploadedFileInfo != nil {
+ break
+ }
+ time.Sleep(1 * time.Second)
+ }
+
+ if fileInfo == nil {
+ logger.Error("Failed to validate manifest file after upload", "id", id)
+ return resp, errors.New("failed to find manifest")
+ }
+
+ if uploadedFileInfo == nil {
+ logger.Error("Failed to validate upload file", "id", id, "manifest", fileInfo, "upload", uploadedFileInfo)
+ return resp, errors.New("failed to find the uploaded file")
+ }
+
+ logger.Info("Upload validated", "id", id, "manifest", fileInfo, "upload_info", uploadedFileInfo)
+ return resp, nil
+}
+
 func getStatusFromError(err error) int {
  if errors.Is(err, info.ErrNotFound) {
  return http.StatusNotFound
@@ -71,7 +117,7 @@ func createInspector(appConfig *appconfig.AppConfig) (UploadInspecter, error) {
  return nil, errors.New("unable to create inspector given app configuration")
 }
 
-func GetUploadInfoHandler(appConfig *appconfig.AppConfig) (http.Handler, error) {
+func GetUploadInfoHandler(appConfig *appconfig.AppConfig) (*InfoHandler, error) {
  inspector, err := createInspector(appConfig)
  return &InfoHandler{
  inspector,

upload-server/cmd/cli/serve.go

@@ -9,9 +9,11 @@ import (
  "github.com/cdcgov/data-exchange-upload/upload-server/internal/handlertusd"
  "github.com/cdcgov/data-exchange-upload/upload-server/internal/health"
  "github.com/cdcgov/data-exchange-upload/upload-server/internal/pshealth"
+ prebuilthooks "github.com/cdcgov/data-exchange-upload/upload-server/pkg/hooks"
  "github.com/cdcgov/data-exchange-upload/upload-server/pkg/redislocker"
  "github.com/prometheus/client_golang/prometheus/promhttp"
  "github.com/tus/tusd/v2/pkg/hooks"
+ tusHooks "github.com/tus/tusd/v2/pkg/hooks"
  "github.com/tus/tusd/v2/pkg/memorylocker"
 )
 
@@ -57,6 +59,9 @@ func Serve(appConfig appconfig.AppConfig) (http.Handler, error) {
  logger.Error("error configuring tusd handler: ", "error", err)
  return nil, err
  }
+ if h, ok := hookHandler.(*prebuilthooks.PrebuiltHook); ok {
+ h.Register(tusHooks.HookPreFinish, uploadInfoHandler.Hook)
+ }
 
  // initialize tusd handler
  handlerTusd, err := handlertusd.New(store, locker, hookHandler, appConfig.TusdHandlerBasePath)

upload-server/cmd/main_test.go

@@ -14,16 +14,16 @@ import (
 )
 
 func TestTus(t *testing.T) {
- url := "http://localhost:8080/files/"
+ url := "http://localhost:8080"
  var wg sync.WaitGroup
  for name, c := range dexTesting.Cases {
  wg.Add(1)
  go func(t *testing.T) {
  defer wg.Done()
- if err := dexTesting.RunTusTestCase(url, "../testing/test/test.txt", c); err != nil {
- t.Error(name, err)
+ if id, err := dexTesting.RunTusTestCase(url, "../testing/test/test.txt", c); err != nil {
+ t.Error(name, err, id)
  } else {
- t.Log("test case", name, "passed")
+ t.Log("test case", name, "passed", id)
  }
  }(t)
  }

upload-server/docker-compose.yml

@@ -28,7 +28,7 @@ services:
  - TUS_AZURE_CONTAINER_NAME=test
  - DEX_MANIFEST_CONFIG_CONTAINER_NAME=config
  - PODMAN_CI_TESTS=true
- - TUS_REDIS_LOCK_URI=redis://redispw@cache:6379
+ - REDIS_CONNECTION_STRING=redis://redispw@cache:6379
  command: go test --tags=integration -coverprofile=c.out -coverpkg=./... ./cmd
  depends_on:
  - azurite