rgw/s3select: json output format for csv, json & parquet

Signed-off-by: Albin Antony <[email protected]>

s3tests_boto3/functional/test_s3select.py

@@ -396,7 +396,7 @@ def test_count_operation():
  obj_to_load = create_random_csv_object(num_of_rows,10)
  upload_object(bucket_name,csv_obj_name,obj_to_load)
  res = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object;") ).replace(",","")
-
+ 
  s3select_assert_result( num_of_rows, int( res ))
 
 @pytest.mark.s3select
@@ -407,16 +407,17 @@ def test_count_json_operation():
  num_of_rows = 1
  obj_to_load = create_random_json_object(num_of_rows,10)
  upload_object(bucket_name,json_obj_name,obj_to_load)
- res = remove_xml_tags_from_result(run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*];"))
- s3select_assert_result( 1, int(res))
+ res = run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*];")
+ s3select_assert_result( '{"_1":1}\n', res)
 
- res = remove_xml_tags_from_result(run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*].root;"))
- s3select_assert_result( 1, int(res))
+ res = run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*].root;")
+ s3select_assert_result( '{"_1":1}\n', res)
 
+ json_obj_name = get_random_string()
  obj_to_load = create_random_json_object(3,10)
  upload_object(bucket_name,json_obj_name,obj_to_load)
- res = remove_xml_tags_from_result(run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*].root;"))
- s3select_assert_result( 3, int(res))
+ res = run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*].root;")
+ s3select_assert_result( '{"_1":3}\n', res)
 
 @pytest.mark.s3select
 def test_json_column_sum_min_max():
@@ -433,58 +434,42 @@ def test_json_column_sum_min_max():
  bucket_name_2 = "testbuck2"
  upload_object(bucket_name_2,json_obj_name_2,json_obj)
 
- res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select min(_1.c1) from s3object[*].root;") ).replace(",","")
+ res_s3select = run_s3select_json(bucket_name,json_obj_name,"select min(_1.c1) from s3object[*].root;")
  list_int = create_list_of_int( 1 , csv_obj )
  res_target = min( list_int )
 
- s3select_assert_result( int(res_s3select), int(res_target))
+ s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))
 
- res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select min(_1.c4) from s3object[*].root;") ).replace(",","")
+ res_s3select = run_s3select_json(bucket_name,json_obj_name,"select min(_1.c4) from s3object[*].root;")
  list_int = create_list_of_int( 4 , csv_obj )
  res_target = min( list_int )
 
- s3select_assert_result( int(res_s3select), int(res_target))
-
- res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select avg(_1.c6) from s3object[*].root;") ).replace(",","")
- list_int = create_list_of_int( 6 , csv_obj )
- res_target = float(sum(list_int ))/10000
-
- s3select_assert_result( float(res_s3select), float(res_target))
+ s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))
 
- res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select max(_1.c4) from s3object[*].root;") ).replace(",","")
+ res_s3select = run_s3select_json(bucket_name,json_obj_name,"select max(_1.c4) from s3object[*].root;")
  list_int = create_list_of_int( 4 , csv_obj )
  res_target = max( list_int )
 
- s3select_assert_result( int(res_s3select), int(res_target))
+ s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))
 
- res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select max(_1.c7) from s3object[*].root;") ).replace(",","")
+ res_s3select = run_s3select_json(bucket_name,json_obj_name,"select max(_1.c7) from s3object[*].root;")
  list_int = create_list_of_int( 7 , csv_obj )
  res_target = max( list_int )
 
- s3select_assert_result( int(res_s3select), int(res_target))
+ s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))
 
- res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select sum(_1.c4) from s3object[*].root;") ).replace(",","")
+ res_s3select = run_s3select_json(bucket_name,json_obj_name,"select sum(_1.c4) from s3object[*].root;")
  list_int = create_list_of_int( 4 , csv_obj )
  res_target = sum( list_int )
 
- s3select_assert_result( int(res_s3select), int(res_target))
+ s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))
 
- res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select sum(_1.c7) from s3object[*].root;") ).replace(",","")
+ res_s3select = run_s3select_json(bucket_name,json_obj_name,"select sum(_1.c7) from s3object[*].root;")
  list_int = create_list_of_int( 7 , csv_obj )
  res_target = sum( list_int )
 
- s3select_assert_result( int(res_s3select) , int(res_target) )
-
- # the following queries, validates on *random* input an *accurate* relation between condition result,sum operation and count operation.
- res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name_2,json_obj_name_2,"select count(0),sum(_1.c1),sum(_1.c2) from s3object[*].root where (_1.c1-_1.c2) = 2;" ) )
- count,sum1,sum2 = res_s3select.split(",")
+ s3select_assert_result( res_s3select, '{{"_1":{}}}\n'.format(res_target))
 
- s3select_assert_result( int(count)*2 , int(sum1)-int(sum2 ) )
-
- res_s3select = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select count(0),sum(_1.c1),sum(_1.c2) from s3object[*].root where (_1.c1-_1.c2) = 4;" ) ) 
- count,sum1,sum2 = res_s3select.split(",")
-
- s3select_assert_result( int(count)*4 , int(sum1)-int(sum2) )
 
 @pytest.mark.s3select
 def test_json_nullif_expressions():