Merge branch 'develop' of github.com:ImageMonkey/imagemonkey-core int…

…o develop

env/docker/Dockerfile.testing

@@ -86,8 +86,9 @@ ENV npm_config_loglevel warn
 # allow installing when the main user is root
 ENV npm_config_unsafe_perm true
 
+# pin cypress to v9.5.4 until https://github.com/cypress-io/cypress/issues/9350 is fixed
 RUN cd /tmp \
-	&& npm install cypress --save-dev \
+	&& npm install cypress@9.5.4 --save-dev \
 	&& npm install --save-dev cypress-file-upload \
 	&& npm install -D cypress-xpath
 

src/api.go

@@ -1662,7 +1662,6 @@ func main() {
 			}
 
 			queryParser := parser.NewQueryParser(query)
-			queryParser.SetVersion(1)
 			parseResult, err := queryParser.Parse()
 			if err != nil {
 				c.JSON(422, gin.H{"error": err.Error()})

src/database/image.go

@@ -616,50 +616,66 @@ func (p *ImageMonkeyDatabase) Export(parseResult parser.ParseResult, annotations
 		identifier = "a.accessor"
 	}
 
-	q := fmt.Sprintf(`SELECT i.key, CASE WHEN json_agg(q3.annotations)::jsonb = '[null]'::jsonb THEN '[]' ELSE json_agg(q3.annotations)::jsonb END as annotations, 
-                      q3.validations, i.width, i.height
-                      FROM image i 
-                      JOIN
-                      (
-                          SELECT COALESCE(q.image_id, q1.image_id) as image_id, q.annotations, q1.validations FROM 
-                          (
-                            SELECT an.image_id as image_id, (d.annotation || ('{"label":"' || %s || '"}')::jsonb || ('{"type":"' || t.name || '"}')::jsonb)::jsonb as annotations 
+	q := fmt.Sprintf(`WITH image_annotation_refinements as (
+						SELECT an.image_id as image_id, (d.annotation || ('{"label":"' || %s || '"}')::jsonb || ('{"type":"' || t.name || '"}')::jsonb)::jsonb as annotations, %s as label
                             FROM image_annotation_refinement r 
                             JOIN annotation_data d ON r.annotation_data_id = d.id
                             JOIN annotation_type t ON d.annotation_type_id = t.id
                             JOIN image_annotation an ON d.image_annotation_id = an.id
                             %s
-                            WHERE ((%s) AND an.auto_generated = false)
-
-                            UNION
-
-                            SELECT n.image_id as image_id, (d.annotation || ('{"label":"' || %s || '"}')::jsonb || ('{"type":"' || t.name || '"}')::jsonb)::jsonb as annotations 
+                            WHERE an.auto_generated = false
+					  ), image_annotations as (
+					 	SELECT n.image_id as image_id, (d.annotation || ('{"label":"' || %s || '"}')::jsonb || ('{"type":"' || t.name || '"}')::jsonb)::jsonb as annotations, %s as label
                             FROM image_annotation n
                             JOIN annotation_data d ON d.image_annotation_id = n.id
                             JOIN annotation_type t ON d.annotation_type_id = t.id
                             %s
-                            WHERE ((%s) AND n.auto_generated = false)
-                          ) q
-                          
-                          %s (
-                            SELECT i.id as image_id, json_agg(json_build_object('label', %s, 'num_yes', num_of_valid, 'num_no', num_of_invalid))::jsonb as validations
+                            WHERE n.auto_generated = false 
+					  ), image_validations as (
+					  	SELECT i.id as image_id, json_agg(json_build_object('label', %s, 'num_yes', num_of_valid, 'num_no', num_of_invalid))::jsonb as validations, array_agg(%s) as accessors
                             FROM image i 
                             JOIN image_validation v ON i.id = v.image_id
                             %s
-                            WHERE (%s)
                             GROUP BY i.id
-                          ) q1 
-                          ON q1.image_id = q.image_id
-                      )q3
-                              
-                     ON i.id = q3.image_id
-                      
-                     WHERE i.unlocked = true
-                     GROUP BY i.key, q3.validations, i.width, i.height`, identifier, q1, parseResult.Query, identifier, q2, parseResult.Query, joinType, identifier, q3, parseResult.Query)
+					  ), unlocked_images as (
+					  	SELECT i.id as image_id, i.key as image_uuid, i.width as image_width, i.height as image_height
+						 FROM image i 
+						 WHERE i.unlocked = true
+					  ), filtered_annotations_and_refinements as (
+					    SELECT image_id, annotations, accessors 
+						FROM (
+							SELECT r.image_id as image_id, r.annotations as annotations, array_agg(label) as accessors
+							FROM image_annotation_refinements r
+							GROUP BY r.image_id, r.annotations
+							
+							UNION
+							
+							SELECT a.image_id as image_id, a.annotations as annotations, array_agg(label) as accessors
+							FROM image_annotations a
+							GROUP BY a.image_id, a.annotations
+						) q
+						WHERE %s
+					  ), filtered_image_validations as (
+					  	SELECT image_id, validations, accessors 
+						FROM image_validations q
+						WHERE %s
+					  )
+
+					  SELECT i.image_uuid, CASE WHEN json_agg(q2.annotations)::jsonb = '[null]'::jsonb THEN '[]' ELSE json_agg(q2.annotations)::jsonb END as annotations
+					  	FROM unlocked_images i
+						JOIN
+						(
+						  SELECT COALESCE(a.image_id, v.image_id) as image_id, a.annotations, v.validations 
+							FROM filtered_annotations_and_refinements a
+							%s
+							filtered_image_validations v
+							ON a.image_id = v.image_id
+						) q2
+						ON q2.image_id = i.image_id
+					    GROUP BY i.image_uuid, q2.validations, i.image_width, i.image_height`, identifier, identifier, q1, identifier, identifier, q2, identifier, identifier, q3, parseResult.Query, parseResult.Query, joinType)
 	rows, err := p.db.Query(context.TODO(), q, parseResult.QueryValues...)
 	if err != nil {
-		log.Debug("Couldn't export data: ", err.Error())
-		log.Error("Export Query: ", q)
+		log.Error("Couldn't export data: ", err.Error())
 		raven.CaptureError(err, nil)
 		return nil, err
 	}

src/database/image_annotation.go

@@ -31,7 +31,9 @@ func MakeAnnotationsProductive(tx pgx.Tx, trendingLabel string, labelId int64) e
 				  				  table_name = 'image_annotation_suggestion' or
 				  				  table_name= 'image_annotation_suggestion_revision' or
 				  				  table_name = 'annotation_suggestion_data' or
-				  				  table_name = 'user_image_annotation_suggestion'
+				  				  table_name = 'user_image_annotation_suggestion' or
+								  table_name = 'image_annotation_refinement' or
+								  table_name = 'image_annotation_suggestion_refinement'
 				  			GROUP BY table_name`)
 	if err != nil {
 		return err
@@ -67,8 +69,12 @@ func MakeAnnotationsProductive(tx pgx.Tx, trendingLabel string, labelId int64) e
 		return errors.New("either the image_annotation_revision or the image_annotation_suggestion_revision table has more columns than expected!")
 	}
 
+	if tableToColumnsMapping["image_annotation_refinement"] != 6 || tableToColumnsMapping["image_annotation_suggestion_refinement"] != 6 {
+		return errors.New("either the image_annotation_refinement or the image_annotation_suggestion_refinement table has more columns than expected!")
+	}
+
 	tempTables := []string{"temp_image_annotation_mapping", "temp_annotation_data_mapping", "temp_image_annotation_revision_mapping",
-		"temp_annotation_data_revision_mapping"}
+		"temp_annotation_data_revision_mapping", "temp_image_annotation_refinement_mapping"}
 
 	for _, tempTable := range tempTables {
 		_, err := tx.Exec(context.TODO(), fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tempTable)) //controlled input, so no sql injection possible
@@ -153,6 +159,23 @@ func MakeAnnotationsProductive(tx pgx.Tx, trendingLabel string, labelId int64) e
 		return err
 	}
 
+
+	_, err = tx.Exec(context.TODO(),
+		`CREATE TEMPORARY TABLE temp_image_annotation_refinement_mapping(old_image_annotation_refinement_id bigint, new_image_annotation_refinement_id bigint,
+										old_annotation_data_id bigint, new_annotation_data_id bigint)`)
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec(context.TODO(),
+		`INSERT INTO temp_image_annotation_refinement_mapping(old_image_annotation_refinement_id, new_image_annotation_refinement_id, old_annotation_data_id, new_annotation_data_id)
+			SELECT r.id, nextval('image_annotation_refinement_id_seq') as new_image_annotation_refinement_id, t.old_annotation_data_id as old_annotation_data_id, t.new_annotation_data_id as new_annotation_data_id 
+			FROM image_annotation_suggestion_refinement r
+			JOIN temp_annotation_data_mapping t ON t.old_annotation_data_id = r.annotation_suggestion_data_id`)
+	if err != nil {
+		return err
+	}
+
 	//insert
 	var insertedImageAnnotationRows int64 = 0
 	err = tx.QueryRow(context.TODO(),
@@ -189,6 +212,21 @@ func MakeAnnotationsProductive(tx pgx.Tx, trendingLabel string, labelId int64) e
 		return err
 	}
 
+	var insertedImageAnnotationRefinementRows int64 = 0
+	err = tx.QueryRow(context.TODO(),
+			`WITH rows AS (
+							 INSERT INTO image_annotation_refinement(id, annotation_data_id, label_id, num_of_valid, sys_period, fingerprint_of_last_modification)
+							 	SELECT new_image_annotation_refinement_id, new_annotation_data_id, r.label_id, r.num_of_valid, r.sys_period, r.fingerprint_of_last_modification
+									FROM temp_image_annotation_refinement_mapping t
+									JOIN image_annotation_suggestion_refinement r ON r.id = t.old_image_annotation_refinement_id
+
+							    RETURNING 1
+						  )
+						  SELECT count(*) FROM rows`).Scan(&insertedImageAnnotationRefinementRows)
+	if err != nil {
+		return err
+	}
+
 	var insertedImageAnnotationRevisionRows int64 = 0
 	err = tx.QueryRow(context.TODO(),
 		`WITH rows AS (
@@ -235,6 +273,23 @@ func MakeAnnotationsProductive(tx pgx.Tx, trendingLabel string, labelId int64) e
 					   SELECT count(*) FROM rows`).Scan(&insertedUserImageAnnotationRows)
 
 	//delete
+	var deletedImageAnnotationSuggestionRefinementRows int64 = 0
+	err = tx.QueryRow(context.TODO(),
+		`WITH rows AS (
+						DELETE FROM image_annotation_suggestion_refinement WHERE id IN
+							(SELECT old_image_annotation_refinement_id FROM temp_image_annotation_refinement_mapping)
+						RETURNING 1
+					  )
+					  SELECT count(*) FROM rows`).Scan(&deletedImageAnnotationSuggestionRefinementRows)
+	if err != nil {
+		return err
+	}
+
+	if insertedImageAnnotationRefinementRows != deletedImageAnnotationSuggestionRefinementRows {
+		return errors.New("inserted image_annotation_refinement rows differ from deleted image_annoation_suggestion_refinment rows!")
+	}
+
+
 	var deletedUserImageAnnotationSuggestionRows int64 = 0
 	err = tx.QueryRow(context.TODO(),
 		`WITH rows AS (

src/parser/v2/parser_test.go

@@ -708,3 +708,10 @@ func TestQueryImageNumAnnotations2(t *testing.T) {
 	equals(t, len(parseResult.QueryValues), 0)
 	equals(t, parseResult.Query, "q.image_num_annotations=1")
 }
+
+func TestQueryCanContainSlash(t *testing.T) {
+	queryParser := NewQueryParser("a/b")
+	parseResult, err := queryParser.Parse()
+	ok(t, err)
+	equals(t, parseResult.Query, "q.accessors @> ARRAY[$1]::text[]")
+}

tests/go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/gin-gonic/gin v1.9.1 // indirect
 	github.com/go-playground/validator/v10 v10.17.0 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
-	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/gomodule/redigo v1.8.9
 	github.com/google/go-jsonnet v0.20.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/jackc/pgconn v1.14.1 // indirect