Skip to content

Commit

Permalink
Optimized AI thingy. MISSING: Body field somehow isn't filling in tim…
Browse files Browse the repository at this point in the history 
…e to be filled by previous runs
  • Loading branch information
@frikky
frikky committed May 13, 2024
1 parent d1aee97 commit b61d927
Show file tree
Hide file tree
Showing 3 changed files with 258 additions and 46 deletions.
15 changes: 8 additions & 7 deletions db-connector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func handleDailyCacheUpdate(executionInfo *ExecutionInfo) *ExecutionInfo {
}
}

log.Printf("[DEBUG] Daily stats not updated for %s in org %s. Only have %d stats so far", timeYesterday, executionInfo.OrgId, len(executionInfo.DailyStatistics))
log.Printf("[DEBUG] Daily stats not updated for %s in org %s today. Only have %d stats so far - running update.", timeYesterday, executionInfo.OrgId, len(executionInfo.DailyStatistics))
// If we get here, we need to update the daily stats
newDay := DailyStatistics{
Date: timeYesterday,
Expand Down Expand Up @@ -3154,7 +3154,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error)
}
}

log.Printf("[INFO] Appending workflows (ADMIN) for organization %s. Already have %d workflows for the user. Found %d (%d new) for org. New unique amount: %d (1)", user.ActiveOrg.Id, userWorkflowLen, len(wrapped.Hits.Hits), len(workflows)-userWorkflowLen, len(workflows))
log.Printf("[INFO] Appending workflows (ADMIN + suborg distribution) for organization %s. Already have %d workflows for the user. Found %d (%d new) for org. New unique amount: %d (1)", user.ActiveOrg.Id, userWorkflowLen, len(wrapped.Hits.Hits), len(workflows)-userWorkflowLen, len(workflows))
}

} else {
Expand Down Expand Up @@ -3217,7 +3217,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error)
}
}

log.Printf("[INFO] Appending suborg distribution workflows for organization %s (%s)", user.ActiveOrg.Name, user.ActiveOrg.Id)
//log.Printf("[INFO] Appending suborg distribution workflows for organization %s (%s)", user.ActiveOrg.Name, user.ActiveOrg.Id)

cursorStr = ""
query = datastore.NewQuery(nameKey).Filter("suborg_distribution =", user.ActiveOrg.Id)
Expand All @@ -3228,10 +3228,12 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error)
innerWorkflow := Workflow{}
_, err := it.Next(&innerWorkflow)
if err != nil {
log.Printf("[ERROR] Error in suborg workflow iterator: %s", err)
if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
log.Printf("[ERROR] Error in workflow loading. Migrating workflow to new workflow handler (1): %s", err)
} else if strings.Contains(fmt.Sprintf("%s", err), "no more items in iterator") {
break
} else {
log.Printf("[ERROR] Error in suborg workflow iterator: %s", err)
break
}
}
Expand All @@ -3251,8 +3253,9 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error)
}
}

// FIXME: Handle nil?
if err != iterator.Done {
log.Printf("[INFO] Failed fetching suborg workflows: %v", err)
//log.Printf("[INFO] Failed fetching suborg workflows: %v", err)
break
}

Expand Down Expand Up @@ -6341,8 +6344,6 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) {
}
}

log.Printf("Found Userapps: %d", len(userApps))

if project.CacheDb {
data, err := json.Marshal(userApps)
if err == nil {
Expand Down
55 changes: 30 additions & 25 deletions files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,8 +432,9 @@ func HandleDeleteFile(resp http.ResponseWriter, request *http.Request) {
}

func LoadStandardFromGithub(client *github.Client, owner, repo, path, filename string) ([]*github.RepositoryContent, error) {
ctx := context.Background()
var err error

ctx := context.Background()
files := []*github.RepositoryContent{}

cacheKey := fmt.Sprintf("github_%s_%s_%s", owner, repo, path)
Expand All @@ -443,36 +444,39 @@ func LoadStandardFromGithub(client *github.Client, owner, repo, path, filename s
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, &files)
if err == nil {
return files, nil
//return files, nil
}
}
}
}

_, items, _, err := client.Repositories.GetContents(ctx, owner, repo, path, nil)
if err != nil {
//log.Printf("[WARNING] Failed getting standard list for namespace %s: %s", path, err)
return files, err
if len(files) == 0 {
_, files, _, err = client.Repositories.GetContents(ctx, owner, repo, path, nil)
if err != nil {
log.Printf("[WARNING] Failed getting standard list for namespace %s: %s", path, err)
return []*github.RepositoryContent{}, err
}
}

if len(items) == 0 {
log.Printf("[WARNING] No items found in namespace %s", path)
return files, errors.New("No items found for namespace")
if len(files) == 0 {
log.Printf("[ERROR] No files found in namespace '%s' on Github - Used for integration framework", path)
return []*github.RepositoryContent{}, nil
}

if len(filename) == 0 {
return items, nil
return []*github.RepositoryContent{}, nil
}

for _, item := range items {
matchingFiles := []*github.RepositoryContent{}
for _, item := range files {
if len(filename) > 0 && strings.HasPrefix(*item.Name, filename) {
files = append(files, item)
matchingFiles = append(matchingFiles, item)
}
}

if project.CacheDb {
data, err := json.Marshal(files)
if err != nil {
log.Printf("[WARNING] Failed marshalling in getfiles: %s", err)
log.Printf("[WARNING] Failed marshalling in get github files: %s", err)
return files, nil
}

Expand All @@ -482,7 +486,7 @@ func LoadStandardFromGithub(client *github.Client, owner, repo, path, filename s
}
}

return files, nil
return matchingFiles, nil
}

func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
Expand Down Expand Up @@ -527,7 +531,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
user.Username = "Execution File API"
}

log.Printf("[AUDIT] User %s (%s) is trying to get files from namespace %#v", user.Username, user.Id, namespace)
log.Printf("[AUDIT] User '%s' (%s) is trying to get files from namespace %#v", user.Username, user.Id, namespace)

ctx := GetContext(request)
files, err := GetAllFiles(ctx, user.ActiveOrg.Id, namespace)
Expand Down Expand Up @@ -592,7 +596,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
// also be environment variables / input arguments
filename, filenameOk := request.URL.Query()["filename"]
if filenameOk && ArrayContains(reservedCategoryNames, namespace) {
log.Printf("[DEBUG] Found name '%s' with reserved category name: %s. Listlength: %d", filename[0], namespace, len(fileResponse.List))
log.Printf("[DEBUG] Filename '%s' in URL with reserved category name: %s. Listlength: %d", filename[0], namespace, len(fileResponse.List))

// Load from Github repo https://github.com/Shuffle/standards
filenameFound := false
Expand All @@ -601,10 +605,11 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
parsedFilename = strings.Replace(parsedFilename, ".json", "", -1)
}

// This is basically a unique handler
for _, item := range fileResponse.List {

itemName := strings.TrimSpace(strings.Replace(strings.ToLower(item.Name), " ", "_", -1))
if itemName == parsedFilename {

if itemName == parsedFilename || itemName == fmt.Sprintf("%s.json", parsedFilename) {
filenameFound = true
break
}
Expand All @@ -628,9 +633,9 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed loading file from Github repo %s/%s"}`, owner, repo)))
//return
} else {
log.Printf("[DEBUG] Found %d file(s) in category %s for filename '%s'", len(foundFiles), namespace, filename[0])
log.Printf("[DEBUG] Found %d file(s) in category '%s' for filename '%s'", len(foundFiles), namespace, filename[0])
for _, item := range foundFiles {
//log.printf("[DEBUG] %#v: %s", *item.Name, item.Status)
log.Printf("[DEBUG] Found file from Github '%s'", *item.Name)

fileContent, _, _, err := client.Repositories.GetContents(ctx, owner, repo, *item.Path, nil)
if err != nil {
Expand All @@ -645,7 +650,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
continue
}

//log.Printf("[DEBUG] Decoded file %s with content:\n%s", *item.Path, string(decoded))
log.Printf("\n\n\n[DEBUG] Decoded file '%s' with content:\n%s\n\n\n", *item.Path, string(decoded))

timeNow := time.Now().Unix()
fileId := "file_"+uuid.NewV4().String()
Expand Down Expand Up @@ -693,7 +698,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
continue
}

log.Printf("[DEBUG] Uploaded file %s with ID %s in category %#v", file.Filename, fileId, namespace)
log.Printf("\n\n[DEBUG] Uploaded file %s with ID %s in category %#v\n\n", file.Filename, fileId, namespace)

fileResponse.List = append(fileResponse.List, BaseFile{
Name: file.Filename,
Expand Down Expand Up @@ -867,15 +872,15 @@ func HandleGetFileContent(resp http.ResponseWriter, request *http.Request) {
user.Username = "Execution File API"
}

log.Printf("[AUDIT] User %s (%s) downloading file %s for org %s", user.Username, user.Id, fileId, user.ActiveOrg.Id)
log.Printf("[AUDIT] User '%s' (%s) downloading file %s in org %s", user.Username, user.Id, fileId, user.ActiveOrg.Id)

// 1. Verify if the user has access to the file: org_id and workflow
ctx := GetContext(request)
file, err := GetFile(ctx, fileId)
if err != nil {
log.Printf("[ERROR] File %s not found: %s", fileId, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
resp.Write([]byte(`{"success": false, "reason": "File not found"}`))
return
}

Expand Down
Loading

0 comments on commit b61d927

Please sign in to comment.