diff --git a/libvmaf/src/feature/third_party/funque/float_funque.c b/libvmaf/src/feature/third_party/funque/float_funque.c index bd220c4b8..2b4c113de 100644 --- a/libvmaf/src/feature/third_party/funque/float_funque.c +++ b/libvmaf/src/feature/third_party/funque/float_funque.c @@ -39,22 +39,37 @@ #include "funque_ssim.h" #include "resizer.h" +#include "funque_strred.h" +#include "funque_strred_options.h" + typedef struct FunqueState { size_t float_stride; float *ref; float *dist; bool debug; + float *pad_ref; + float *pad_dist; VmafPicture res_ref_pic; VmafPicture res_dist_pic; + int spatial_csf_filter; + int wavelet_csf_filter; + char *spatial_csf_filter_type; + char *wavelet_csf_filter_type; + float *spat_tmp_buf; size_t float_dwt2_stride; float *spat_filter; float csf_factors[4][4]; dwt2buffers ref_dwt2out[4]; dwt2buffers dist_dwt2out[4]; + strredbuffers prev_ref[4]; + strredbuffers prev_dist[4]; + strred_results strred_scores; // funque configurable parameters + //const char *wavelet_csfs; + bool enable_resize; bool enable_spatial_csf; int vif_levels; @@ -62,8 +77,14 @@ typedef struct FunqueState { int needed_dwt_levels; int needed_full_dwt_levels; int ssim_levels; + int ms_ssim_levels; double norm_view_dist; int ref_display_height; + int strred_levels; + int process_ref_width; + int process_ref_height; + int process_dist_width; + int process_dist_height; // VIF extra variables double vif_enhn_gain_limit; @@ -75,6 +96,8 @@ typedef struct FunqueState { VmafDictionary *feature_name_dict; ResizerState resize_module; + MsSsimScore *score; + } FunqueState; static const VmafOption options[] = { @@ -99,8 +122,27 @@ static const VmafOption options[] = { .help = "enable the global CSF based on spatial filter", .offset = offsetof(FunqueState, enable_spatial_csf), .type = VMAF_OPT_TYPE_BOOL, - .default_val.b = true, - .flags = VMAF_OPT_FLAG_FEATURE_PARAM, + .default_val.b = true + }, + { + .name = "spatial_csf_filter", + .alias = "spatial_csf_filter", + .help = "Select number of taps to be used for spatial filter", + .offset = offsetof(FunqueState, spatial_csf_filter), + .type = VMAF_OPT_TYPE_INT, + .default_val.i = NADENAU_SPAT_5_TAP_FILTER, + .min = NADENAU_SPAT_5_TAP_FILTER, + .max = NGAN_21_TAP_FILTER, + }, + { + .name = "wavelet_csf_filter", + .alias = "wave_filter", + .help = "Select wavelet filter", + .offset = offsetof(FunqueState, wavelet_csf_filter), + .type = VMAF_OPT_TYPE_INT, + .default_val.i = NADENAU_WEIGHT_FILTER, + .min = NADENAU_WEIGHT_FILTER, + .max = MANNOS_WEIGHT_FILTER, }, { .name = "norm_view_dist", @@ -144,6 +186,16 @@ static const VmafOption options[] = { .min = MIN_LEVELS, .max = MAX_LEVELS, }, + { + .name = "ms_ssim_levels", + .alias = "ms_ssiml", + .help = "Number of DWT levels for MS_SSIM", + .offset = offsetof(FunqueState, ms_ssim_levels), + .type = VMAF_OPT_TYPE_INT, + .default_val.i = DEFAULT_MS_SSIM_LEVELS, + .min = MIN_LEVELS, + .max = MAX_LEVELS, + }, { .name = "vif_enhn_gain_limit", .alias = "egl", @@ -190,22 +242,30 @@ static const VmafOption options[] = { .max = DEFAULT_ADM_ENHN_GAIN_LIMIT, .flags = VMAF_OPT_FLAG_FEATURE_PARAM, }, + { + .name = "strred_levels", + .alias = "strred", + .help = "Number of levels in STRRED", + .offset = offsetof(FunqueState, strred_levels), + .type = VMAF_OPT_TYPE_INT, + .default_val.i = DEFAULT_STRRED_LEVELS, + .min = MIN_LEVELS, + .max = MAX_LEVELS, + }, { 0 } }; static int alloc_dwt2buffers(dwt2buffers *dwt2out, int w, int h) { - dwt2out->width = (int) (w+1)/2; - dwt2out->height = (int) (h+1)/2; - dwt2out->stride = ALIGN_CEIL(dwt2out->width * sizeof(float)); + dwt2out->width = (int) w; + dwt2out->height = (int) h; + dwt2out->stride = dwt2out->width * sizeof(float); for(unsigned i=0; i<4; i++) { dwt2out->bands[i] = aligned_malloc(dwt2out->stride * dwt2out->height, 32); if (!dwt2out->bands[i]) goto fail; - - dwt2out->bands[i] = aligned_malloc(dwt2out->stride * dwt2out->height, 32); - if (!dwt2out->bands[i]) goto fail; + memset(dwt2out->bands[i], 0, dwt2out->stride * dwt2out->height); } return 0; @@ -217,10 +277,51 @@ static int alloc_dwt2buffers(dwt2buffers *dwt2out, int w, int h) { return -ENOMEM; } +void select_filter_type(FunqueState *s) +{ + if (s->enable_spatial_csf == 1) + { + if (s->spatial_csf_filter == 5) + s->spatial_csf_filter_type = "nadenau_spat"; + else if (s->spatial_csf_filter == 21) + s->spatial_csf_filter_type = "ngan_spat"; + } + else + { + switch(s->wavelet_csf_filter) + { + case 1: + s->wavelet_csf_filter_type = "nadenau_weight"; + break; + + case 2: + s->wavelet_csf_filter_type = "li"; + break; + + case 3: + s->wavelet_csf_filter_type = "hill"; + break; + + case 4: + s->wavelet_csf_filter_type = "watson"; + break; + + case 5: + s->wavelet_csf_filter_type = "mannos_weight"; + break; + + default: + s->wavelet_csf_filter_type = "nadenau_weight"; + break; + } + } +} + static int init(VmafFeatureExtractor *fex, enum VmafPixelFormat pix_fmt, unsigned bpc, unsigned w, unsigned h) { (void)pix_fmt; + (void)bpc; FunqueState *s = fex->priv; s->feature_name_dict = @@ -234,49 +335,159 @@ static int init(VmafFeatureExtractor *fex, enum VmafPixelFormat pix_fmt, h = (h+1)>>1; } - s->needed_dwt_levels = MAX(MAX(s->vif_levels, s->adm_levels), s->ssim_levels); + s->needed_dwt_levels = MAX5(s->vif_levels, s->adm_levels, s->ssim_levels, s->ms_ssim_levels, s->strred_levels); s->needed_full_dwt_levels = MAX(s->adm_levels, s->ssim_levels); - s->float_stride = ALIGN_CEIL(w * sizeof(float)); + int ref_process_width, ref_process_height, dist_process_width, dist_process_height, process_wh_div_factor; + + int last_w = w; + int last_h = h; + + if(s->ms_ssim_levels != 0){ +#if ENABLE_PADDING + int two_pow_level_m1 = pow(2, (s->needed_dwt_levels - 1)); + ref_process_width = (int) (((last_w + two_pow_level_m1) >> s->needed_dwt_levels) << s->needed_dwt_levels); + ref_process_height = (int) (((last_h + two_pow_level_m1) >> s->needed_dwt_levels) << s->needed_dwt_levels); + dist_process_width = (int) (((last_w + two_pow_level_m1) >> s->needed_dwt_levels) << s->needed_dwt_levels); + dist_process_height = (int) (((last_h + two_pow_level_m1) >> s->needed_dwt_levels) << s->needed_dwt_levels); + +#else // Cropped width and height + ref_process_width = (int) ((last_w >> s->needed_dwt_levels) << s->needed_dwt_levels); + ref_process_height = (int) ((last_h >> s->needed_dwt_levels) << s->needed_dwt_levels); + dist_process_width = (int) ((last_w >> s->needed_dwt_levels) << s->needed_dwt_levels); + dist_process_height = (int) ((last_h >> s->needed_dwt_levels) << s->needed_dwt_levels); +#endif + + last_w = ref_process_width; + last_h = ref_process_height; + } + else + { + ref_process_width = last_w; + ref_process_height = last_h; + dist_process_width = last_w; + dist_process_height = last_h; + } + + s->float_stride = ALIGN_CEIL(ref_process_width * sizeof(float)); if(s->enable_resize) { - s->res_ref_pic.data[0] = aligned_malloc(s->float_stride * h, 32); + s->res_ref_pic.data[0] = aligned_malloc(s->float_stride * ref_process_height, 32); if (!s->res_ref_pic.data[0]) goto fail; - s->res_dist_pic.data[0] = aligned_malloc(s->float_stride * h, 32); + memset(s->res_ref_pic.data[0], 0, s->float_stride * ref_process_height); + + s->res_dist_pic.data[0] = aligned_malloc(s->float_stride * ref_process_height, 32); if (!s->res_dist_pic.data[0]) goto fail; + memset(s->res_dist_pic.data[0], 0, s->float_stride * ref_process_height); } - s->ref = aligned_malloc(s->float_stride * h, 32); + s->ref = aligned_malloc(s->float_stride * ref_process_height, 32); if (!s->ref) goto fail; - s->dist = aligned_malloc(s->float_stride * h, 32); + memset(s->ref, 0, s->float_stride * ref_process_height); + + s->dist = aligned_malloc(s->float_stride * ref_process_height, 32); if (!s->dist) goto fail; + memset(s->dist, 0, s->float_stride * ref_process_height); + +#if ENABLE_PADDING + s->pad_ref = aligned_malloc(s->float_stride * ref_process_height, 32); + if (!s->pad_ref) goto fail; + memset(s->pad_ref, 0, s->float_stride * ref_process_height); + + s->pad_dist = aligned_malloc(s->float_stride * dist_process_height, 32); + if (!s->pad_dist) goto fail; + memset(s->pad_dist, 0, s->float_stride * dist_process_height); +#endif + + select_filter_type(s); if (s->enable_spatial_csf) { - s->spat_filter = aligned_malloc(s->float_stride * h, 32); - if (!s->spat_filter) - goto fail; + s->spat_tmp_buf = aligned_malloc(s->float_stride, 32); + if (!s->spat_tmp_buf) goto fail; + memset(s->spat_tmp_buf, 0, s->float_stride); + + s->spat_filter = aligned_malloc(s->float_stride * ref_process_height, 32); + if (!s->spat_filter) goto fail; + memset(s->spat_filter, 0, s->float_stride * ref_process_height); + + } else { + if(strcmp(s->wavelet_csf_filter_type, "nadenau_weight") == 0) { + for(int level = 0; level < 4; level++) { + s->csf_factors[level][0] = nadenau_weight_coeffs[level][0]; + s->csf_factors[level][1] = nadenau_weight_coeffs[level][1]; + s->csf_factors[level][2] = nadenau_weight_coeffs[level][2]; + s->csf_factors[level][3] = nadenau_weight_coeffs[level][3]; + } + } else if(strcmp(s->wavelet_csf_filter_type, "li") == 0) { + for(int level = 0; level < 4; level++) { + s->csf_factors[level][0] = li_coeffs[level][0]; + s->csf_factors[level][1] = li_coeffs[level][1]; + s->csf_factors[level][2] = li_coeffs[level][2]; + s->csf_factors[level][3] = li_coeffs[level][3]; + } + } else if(strcmp(s->wavelet_csf_filter_type, "hill") == 0) { + for(int level = 0; level < 4; level++) { + s->csf_factors[level][0] = hill_coeffs[level][0]; + s->csf_factors[level][1] = hill_coeffs[level][1]; + s->csf_factors[level][2] = hill_coeffs[level][2]; + s->csf_factors[level][3] = hill_coeffs[level][3]; + } + } else if(strcmp(s->wavelet_csf_filter_type, "watson") == 0) { + for(int level = 0; level < 4; level++) { + s->csf_factors[level][0] = watson_coeffs[level][0]; + s->csf_factors[level][1] = watson_coeffs[level][1]; + s->csf_factors[level][2] = watson_coeffs[level][2]; + s->csf_factors[level][3] = watson_coeffs[level][3]; + } + } else if(strcmp(s->wavelet_csf_filter_type, "mannos_weight") == 0) { + for(int level = 0; level < 4; level++) { + s->csf_factors[level][0] = mannos_weight_coeffs[level][0]; + s->csf_factors[level][1] = mannos_weight_coeffs[level][1]; + s->csf_factors[level][2] = mannos_weight_coeffs[level][2]; + s->csf_factors[level][3] = mannos_weight_coeffs[level][3]; + } } else { for(int level = 0; level < 4; level++) { - s->csf_factors[level][0] = 1.0f / funque_dwt_quant_step(&funque_dwt_7_9_YCbCr_threshold[0], level, 0, s->norm_view_dist, s->ref_display_height); - s->csf_factors[level][1] = 1.0f / funque_dwt_quant_step(&funque_dwt_7_9_YCbCr_threshold[0], level, 1, s->norm_view_dist, s->ref_display_height); - s->csf_factors[level][2] = 1.0f / funque_dwt_quant_step(&funque_dwt_7_9_YCbCr_threshold[0], level, 2, s->norm_view_dist, s->ref_display_height); + s->csf_factors[level][0] = + 1.0f / funque_dwt_quant_step(&funque_dwt_7_9_YCbCr_threshold[0], level, 0, + s->norm_view_dist, s->ref_display_height); + s->csf_factors[level][1] = + 1.0f / funque_dwt_quant_step(&funque_dwt_7_9_YCbCr_threshold[0], level, 1, + s->norm_view_dist, s->ref_display_height); + s->csf_factors[level][2] = + 1.0f / funque_dwt_quant_step(&funque_dwt_7_9_YCbCr_threshold[0], level, 2, + s->norm_view_dist, s->ref_display_height); s->csf_factors[level][3] = s->csf_factors[level][1]; // same as horizontal } } + } int err = 0; - - int last_w = w; - int last_h = h; + int tref_width, tref_height, tdist_width, tdist_height; for (int level = 0; level < s->needed_dwt_levels; level++) { - err |= alloc_dwt2buffers(&s->ref_dwt2out[level], last_w, last_h); - err |= alloc_dwt2buffers(&s->dist_dwt2out[level], last_w, last_h); - last_w = s->ref_dwt2out[level].width; - last_h = s->ref_dwt2out[level].height; + process_wh_div_factor = pow(2, (level+1)); + tref_width = (ref_process_width + (process_wh_div_factor * 3/4)) / process_wh_div_factor; + tref_height = (ref_process_height + (process_wh_div_factor * 3/4)) / process_wh_div_factor; + tdist_width = (dist_process_width + (process_wh_div_factor * 3/4)) / process_wh_div_factor; + tdist_height = (dist_process_height + (process_wh_div_factor * 3/4)) / process_wh_div_factor; + + err |= alloc_dwt2buffers(&s->ref_dwt2out[level], tref_width, tref_height); + err |= alloc_dwt2buffers(&s->dist_dwt2out[level], tdist_width, tdist_height); + + s->prev_ref[level].bands[0] = NULL; + s->prev_dist[level].bands[0] = NULL; + + for(int subband = 1; subband < 4; subband++) { + s->prev_ref[level].bands[subband] = (float*) calloc(tref_width * tref_height, sizeof(float)); + s->prev_dist[level].bands[subband] = (float*) calloc(tref_width * tref_height, sizeof(float)); + } + + last_w = (int) (last_w + 1) / 2; + last_h = (int) (last_h + 1) / 2; } if (err) goto fail; @@ -290,13 +501,16 @@ static int init(VmafFeatureExtractor *fex, enum VmafPixelFormat pix_fmt, if (s->res_dist_pic.data[0]) aligned_free(s->res_dist_pic.data[0]); if (s->ref) aligned_free(s->ref); if (s->dist) aligned_free(s->dist); +#if ENABLE_PADDING + if (s->pad_ref) aligned_free(s->pad_ref); + if (s->pad_dist) aligned_free(s->pad_dist); +#endif if (s->spat_filter) aligned_free(s->spat_filter); + if (s->spat_tmp_buf) aligned_free(s->spat_tmp_buf); vmaf_dictionary_free(&s->feature_name_dict); return -ENOMEM; } -// #define MIN(x, y) (((x) < (y)) ? (x) : (y)) - static int extract(VmafFeatureExtractor *fex, VmafPicture *ref_pic, VmafPicture *ref_pic_90, VmafPicture *dist_pic, VmafPicture *dist_pic_90, @@ -343,40 +557,113 @@ static int extract(VmafFeatureExtractor *fex, res_dist_pic = dist_pic; } - funque_picture_copy(s->ref, s->float_stride, res_ref_pic, 0, ref_pic->bpc); - funque_picture_copy(s->dist, s->float_stride, res_dist_pic, 0, dist_pic->bpc); + + if(s->ms_ssim_levels != 0){ +#if ENABLE_PADDING + int two_pow_level_m1 = pow(2, (s->needed_dwt_levels - 1)); + s->process_ref_width = ((res_ref_pic->w[0] + two_pow_level_m1) >> s->needed_dwt_levels) << s->needed_dwt_levels; + s->process_ref_height = ((res_ref_pic->h[0] + two_pow_level_m1) >> s->needed_dwt_levels) << s->needed_dwt_levels; + s->process_dist_width = ((res_dist_pic->w[0] + two_pow_level_m1) >> s->needed_dwt_levels) << s->needed_dwt_levels; + s->process_dist_height = ((res_dist_pic->h[0] + two_pow_level_m1) >> s->needed_dwt_levels) << s->needed_dwt_levels; +#else + s->process_ref_width = (res_ref_pic->w[0] >> s->needed_dwt_levels) << s->needed_dwt_levels; + s->process_ref_height = (res_ref_pic->h[0] >> s->needed_dwt_levels) << s->needed_dwt_levels; + s->process_dist_width = (res_dist_pic->w[0] >> s->needed_dwt_levels) << s->needed_dwt_levels; + s->process_dist_height = (res_dist_pic->h[0] >> s->needed_dwt_levels) << s->needed_dwt_levels; +#endif + } + else + { + s->process_ref_width = res_ref_pic->w[0]; + s->process_ref_height = res_ref_pic->h[0]; + s->process_dist_width = res_dist_pic->w[0]; + s->process_dist_height = res_dist_pic->h[0]; + } + +#if ENABLE_PADDING + funque_picture_copy(s->ref, s->float_stride, res_ref_pic, 0, ref_pic->bpc, res_ref_pic->w[0], res_ref_pic->h[0]); + funque_picture_copy(s->dist, s->float_stride, res_dist_pic, 0, dist_pic->bpc, res_dist_pic->w[0], res_dist_pic->h[0]); int bitdepth_pow2 = (1 << res_ref_pic->bpc) - 1; - normalize_bitdepth(s->ref, s->ref, bitdepth_pow2, s->float_stride, res_ref_pic->w[0], res_ref_pic->h[0]); - normalize_bitdepth(s->dist, s->dist, bitdepth_pow2, s->float_stride, res_dist_pic->w[0], res_dist_pic->h[0]); + normalize_bitdepth(s->ref, s->ref, bitdepth_pow2, s->float_stride, s->process_ref_width, s->process_ref_height); + normalize_bitdepth(s->dist, s->dist, bitdepth_pow2, s->float_stride, s->process_dist_width, s->process_dist_height); + + int reflect_width, reflect_height; + reflect_width = (s->process_ref_width - res_ref_pic->w[0]) / 2; + reflect_height = (s->process_ref_height - res_ref_pic->h[0]) / 2; + reflect_pad_for_input(s->ref, s->pad_ref, res_ref_pic->w[0], res_ref_pic->h[0], reflect_width, reflect_height); + + reflect_width = (s->process_dist_width - res_dist_pic->w[0]) / 2; + reflect_height = (s->process_dist_height - res_dist_pic->h[0]) / 2; + reflect_pad_for_input(s->dist, s->pad_dist, res_dist_pic->w[0], res_dist_pic->h[0], reflect_width, reflect_height); if (s->enable_spatial_csf) { - spatial_filter(s->ref, s->spat_filter, res_ref_pic->w[0], res_ref_pic->h[0]); - funque_dwt2(s->spat_filter, &s->ref_dwt2out[0], res_ref_pic->w[0], res_ref_pic->h[0]); - spatial_filter(s->dist, s->spat_filter, res_dist_pic->w[0], res_dist_pic->h[0]); - funque_dwt2(s->spat_filter, &s->dist_dwt2out[0], res_dist_pic->w[0], res_dist_pic->h[0]); + /*assume this is entering the path of FullScaleY Funque Extractor*/ + /*CSF factors are applied to the pictures based on predefined thresholds.*/ + spatial_csfs(s->pad_ref, s->spat_filter, s->process_ref_width, s->process_ref_height, s->spat_tmp_buf, s->spatial_csf_filter_type); + funque_dwt2(s->spat_filter, &s->ref_dwt2out[0], s->process_ref_width, s->process_ref_height); + spatial_csfs(s->pad_dist, s->spat_filter, s->process_dist_width, s->process_dist_height, s->spat_tmp_buf, s->spatial_csf_filter_type); + funque_dwt2(s->spat_filter, &s->dist_dwt2out[0], s->process_dist_width, s->process_dist_height); + } else { - funque_dwt2(s->ref, &s->ref_dwt2out[0], res_ref_pic->w[0], res_ref_pic->h[0]); - funque_dwt2(s->dist, &s->dist_dwt2out[0], res_dist_pic->w[0], res_dist_pic->h[0]); + // Wavelet Domain or pyramid is done + funque_dwt2(s->pad_ref, &s->ref_dwt2out[0], s->process_ref_width, s->process_ref_height); + funque_dwt2(s->pad_dist, &s->dist_dwt2out[0], s->process_dist_width, s->process_dist_height); } +#else + funque_picture_copy(s->ref, s->float_stride, res_ref_pic, 0, ref_pic->bpc, s->process_ref_width, s->process_ref_height); + funque_picture_copy(s->dist, s->float_stride, res_dist_pic, 0, dist_pic->bpc, s->process_dist_width, s->process_dist_height); + + int bitdepth_pow2 = (1 << res_ref_pic->bpc) - 1; + normalize_bitdepth(s->ref, s->ref, bitdepth_pow2, s->float_stride, s->process_ref_width, s->process_ref_height); + normalize_bitdepth(s->dist, s->dist, bitdepth_pow2, s->float_stride, s->process_dist_width, s->process_dist_height); + + if (s->enable_spatial_csf) { + /*assume this is entering the path of FullScaleY Funque Extractor*/ + /*CSF factors are applied to the pictures based on predefined thresholds.*/ + spatial_csfs(s->ref, s->spat_filter, s->process_ref_width, s->process_ref_height, s->spat_tmp_buf, s->spatial_csf_filter_type); + funque_dwt2(s->spat_filter, &s->ref_dwt2out[0], s->process_ref_width, s->process_ref_height); + spatial_csfs(s->dist, s->spat_filter, s->process_dist_width, s->process_dist_height, s->spat_tmp_buf, s->spatial_csf_filter_type); + funque_dwt2(s->spat_filter, &s->dist_dwt2out[0], s->process_dist_width, s->process_dist_height); + + } else { + // Wavelet Domain or pyramid is done + funque_dwt2(s->ref, &s->ref_dwt2out[0], s->process_ref_width, s->process_ref_height); + funque_dwt2(s->dist, &s->dist_dwt2out[0], s->process_dist_width, s->process_dist_height); + } +#endif double ssim_score[MAX_LEVELS]; + MsSsimScore ms_ssim_score[MAX_LEVELS]; + s->score = ms_ssim_score; double adm_score[MAX_LEVELS], adm_score_num[MAX_LEVELS], adm_score_den[MAX_LEVELS]; double vif_score[MAX_LEVELS], vif_score_num[MAX_LEVELS], vif_score_den[MAX_LEVELS]; + float *var_x_cum = (float *) calloc(res_ref_pic->w[0] * res_ref_pic->h[0], sizeof(float)); + float *var_y_cum = (float *) calloc(res_ref_pic->w[0] * res_ref_pic->h[0], sizeof(float)); + float *cov_xy_cum = (float *) calloc(res_ref_pic->w[0] * res_ref_pic->h[0], sizeof(float)); + + ms_ssim_score[0].var_x_cum = &var_x_cum; + ms_ssim_score[0].var_y_cum = &var_y_cum; + ms_ssim_score[0].cov_xy_cum = &cov_xy_cum; + double adm_den = 0.0; double adm_num = 0.0; double vif_den = 0.0; double vif_num = 0.0; + s->strred_scores.spat_vals_cumsum = 0; + s->strred_scores.temp_vals_cumsum = 0; + s->strred_scores.spat_temp_vals_cumsum = 0; + for (int level = 0; level < s->needed_dwt_levels; level++) { // pre-compute the next level of DWT if (level+1 < s->needed_dwt_levels) { if (level+1 > s->needed_full_dwt_levels - 1) { // from here on out we only need approx band for VIF - funque_vifdwt2_band0(s->ref_dwt2out[level].bands[0], s->ref_dwt2out[level + 1].bands[0], s->ref_dwt2out[level].stride, s->ref_dwt2out[level].width, s->ref_dwt2out[level].height); + funque_vifdwt2_band0(s->ref_dwt2out[level].bands[0], s->ref_dwt2out[level + 1].bands[0], s->ref_dwt2out[level].width, s->ref_dwt2out[level].width, s->ref_dwt2out[level].height); } else { // compute full DWT if either SSIM or ADM need it for this level funque_dwt2(s->ref_dwt2out[level].bands[0], &s->ref_dwt2out[level + 1], s->ref_dwt2out[level].width, @@ -398,17 +685,31 @@ static int extract(VmafFeatureExtractor *fex, } } - if (level <= s->adm_levels - 1) { + if ((s->adm_levels != 0) && (level <= s->adm_levels - 1)) { err |= compute_adm_funque(s->ref_dwt2out[level], s->dist_dwt2out[level], &adm_score[level], &adm_score_num[level], &adm_score_den[level], ADM_BORDER_FACTOR); adm_num += adm_score_num[level]; adm_den += adm_score_den[level]; } - if (level <= s->ssim_levels - 1) { + if ((s->ssim_levels != 0) && (level <= s->ssim_levels - 1)) { err |= compute_ssim_funque(&s->ref_dwt2out[level], &s->dist_dwt2out[level], &ssim_score[level], 1, (float)0.01, (float)0.03); } - if (level <= s->vif_levels - 1) { + if((s->ms_ssim_levels != 0) && (level <= s->ms_ssim_levels - 1)) { + err |= compute_ms_ssim_funque(&s->ref_dwt2out[level], &s->dist_dwt2out[level], + &ms_ssim_score[level], 1, (float) 0.01, (float) 0.03, + (level + 1)); + + err |= mean_2x2_ms_ssim_funque(var_x_cum, var_y_cum, cov_xy_cum, s->ref_dwt2out[level].width, s->ref_dwt2out[level].height, level); + + if(level != s->ms_ssim_levels - 1) { + ms_ssim_score[level + 1].var_x_cum = ms_ssim_score[level].var_x_cum; + ms_ssim_score[level + 1].var_y_cum = ms_ssim_score[level].var_y_cum; + ms_ssim_score[level + 1].cov_xy_cum = ms_ssim_score[level].cov_xy_cum; + } + } + + if ((s->vif_levels != 0) && (level <= s->vif_levels - 1)) { #if USE_DYNAMIC_SIGMA_NSQ err |= compute_vif_funque(s->ref_dwt2out[level].bands[0], s->dist_dwt2out[level].bands[0], s->ref_dwt2out[level].width, s->ref_dwt2out[level].height, &vif_score[level], &vif_score_num[level], &vif_score_den[level], VIF_WINDOW_SIZE, 1, (double)VIF_SIGMA_NSQ, level); @@ -420,12 +721,37 @@ static int extract(VmafFeatureExtractor *fex, vif_den += vif_score_den[level]; } + if((s->strred_levels != 0) && (level <= s->strred_levels - 1)) { + if(index == 0) { + err |= copy_prev_frame_strred_funque( + &s->ref_dwt2out[level], &s->dist_dwt2out[level], &s->prev_ref[level], + &s->prev_dist[level], s->ref_dwt2out[level].width, + s->ref_dwt2out[level].height); + } + else { + err |= compute_strred_funque( + &s->ref_dwt2out[level], &s->dist_dwt2out[level], &s->prev_ref[level], + &s->prev_dist[level], s->ref_dwt2out[level].width, s->ref_dwt2out[level].height, + &s->strred_scores, BLOCK_SIZE, level); + + err |= copy_prev_frame_strred_funque( + &s->ref_dwt2out[level], &s->dist_dwt2out[level], &s->prev_ref[level], + &s->prev_dist[level], s->ref_dwt2out[level].width, + s->ref_dwt2out[level].height); + } + } + if (err) return err; } + if(s->ms_ssim_levels != 0) { + err |= compute_ms_ssim_mean_scales(ms_ssim_score, s->ssim_levels); + } + double vif = vif_den > 0 ? vif_num / vif_den : 1.0; double adm = adm_den > 0 ? adm_num / adm_den : 1.0; +if (s->vif_levels > 0) { err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, "FUNQUE_feature_vif_score", vif, index); @@ -451,7 +777,9 @@ static int extract(VmafFeatureExtractor *fex, } } } +} +if (s->adm_levels > 0) { err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, "FUNQUE_feature_adm_score", adm, index); @@ -477,8 +805,11 @@ static int extract(VmafFeatureExtractor *fex, } } } +} - err |= vmaf_feature_collector_append(feature_collector, "FUNQUE_feature_ssim_scale0_score", +if (s->ssim_levels > 0) { + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_ssim_scale0_score", ssim_score[0], index); if (s->ssim_levels > 1) { @@ -498,6 +829,115 @@ static int extract(VmafFeatureExtractor *fex, } } } +} + +if(s->strred_levels > 0) { + if(index == 0) { + err |= + vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_strred_scale0_score", 0, index); + + if(s->strred_levels > 1) { + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_strred_scale1_score", 0, + index); + + if(s->strred_levels > 2) { + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, "FUNQUE_feature_strred_scale2_score", + 0, index); + + if(s->strred_levels > 3) { + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, + "FUNQUE_feature_strred_scale3_score", 0, index); + } + } + } + } else { + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_strred_scale0_score", + s->strred_scores.strred_vals[0], index); + + if(s->strred_levels > 1) { + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_strred_scale1_score", + s->strred_scores.strred_vals[1], index); + + if(s->strred_levels > 2) { + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, "FUNQUE_feature_strred_scale2_score", + s->strred_scores.strred_vals[2], index); + + if(s->strred_levels > 3) { + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, + "FUNQUE_feature_strred_scale3_score", s->strred_scores.strred_vals[3], + index); + } + } + } + } +} + +if(s->ms_ssim_levels > 0) { + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_ms_ssim_mean_scale0_score", + s->score[0].ms_ssim_mean, index); + + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_ms_ssim_cov_scale0_score", + s->score[0].ms_ssim_cov, index); + + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_ms_ssim_mink3_scale0_score", + s->score[0].ms_ssim_mink3, index); + + if(s->ms_ssim_levels > 1) { + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_ms_ssim_mean_scale1_score", + s->score[1].ms_ssim_mean, index); + + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_ms_ssim_cov_scale1_score", + s->score[1].ms_ssim_cov, index); + + err |= vmaf_feature_collector_append_with_dict(feature_collector, s->feature_name_dict, + "FUNQUE_feature_ms_ssim_mink3_scale1_score", + s->score[1].ms_ssim_mink3, index); + + if(s->ms_ssim_levels > 2) { + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, "FUNQUE_feature_ms_ssim_mean_scale2_score", + s->score[2].ms_ssim_mean, index); + + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, "FUNQUE_feature_ms_ssim_cov_scale2_score", + s->score[2].ms_ssim_cov, index); + + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, "FUNQUE_feature_ms_ssim_mink3_scale2_score", + s->score[2].ms_ssim_mink3, index); + + if(s->ms_ssim_levels > 3) { + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, + "FUNQUE_feature_ms_ssim_mean_scale3_score", s->score[3].ms_ssim_mean, index); + + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, + "FUNQUE_feature_ms_ssim_cov_scale3_score", s->score[3].ms_ssim_cov, index); + err |= vmaf_feature_collector_append_with_dict( + feature_collector, s->feature_name_dict, + "FUNQUE_feature_ms_ssim_mink3_scale3_score", s->score[3].ms_ssim_mink3, index); + } + } + } +} + + free(var_x_cum); + free(var_y_cum); + free(cov_xy_cum); return err; } @@ -509,7 +949,13 @@ static int close(VmafFeatureExtractor *fex) if (s->res_dist_pic.data[0]) aligned_free(s->res_dist_pic.data[0]); if (s->ref) aligned_free(s->ref); if (s->dist) aligned_free(s->dist); +#if ENABLE_PADDING + if (s->pad_ref) aligned_free(s->pad_ref); + if (s->pad_dist) aligned_free(s->pad_dist); +#endif if (s->spat_filter) aligned_free(s->spat_filter); + if (s->spat_tmp_buf) aligned_free(s->spat_tmp_buf); + for(int level = 0; level < s->needed_dwt_levels; level += 1) { for(unsigned i=0; i<4; i++) @@ -517,6 +963,11 @@ static int close(VmafFeatureExtractor *fex) if (s->ref_dwt2out[level].bands[i]) aligned_free(s->ref_dwt2out[level].bands[i]); if (s->dist_dwt2out[level].bands[i]) aligned_free(s->dist_dwt2out[level].bands[i]); } + for(unsigned i=1; i<4; i++) + { + if (s->prev_ref[level].bands[i]) free(s->prev_ref[level].bands[i]); + if (s->prev_dist[level].bands[i]) free(s->prev_dist[level].bands[i]); + } } vmaf_dictionary_free(&s->feature_name_dict); @@ -525,14 +976,25 @@ static int close(VmafFeatureExtractor *fex) static const char *provided_features[] = { "FUNQUE_feature_vif_score", - "FUNQUE_feature_vif_scale0", "FUNQUE_feature_vif_scale1_score", - "FUNQUE_feature_vif_scale2", "FUNQUE_feature_vif_scale3_score", + "FUNQUE_feature_vif_scale0_score", "FUNQUE_feature_vif_scale1_score", + "FUNQUE_feature_vif_scale2_score", "FUNQUE_feature_vif_scale3_score", + + "FUNQUE_feature_adm_score", + "FUNQUE_feature_adm_scale0_score","FUNQUE_feature_adm_scale1_score", + "FUNQUE_feature_adm_scale2_score","FUNQUE_feature_adm_scale3_score", + + "FUNQUE_feature_ssim_scale0_score", "FUNQUE_feature_ssim_scale1_score", + "FUNQUE_feature_ssim_scale2_score", "FUNQUE_feature_ssim_scale3_score", - "FUNQUE_feature_adm2_score", - "FUNQUE_feature_adm_scale0","FUNQUE_feature_adm_scale1_score", - "FUNQUE_feature_adm_scale2","FUNQUE_feature_adm_scale3_score", + "FUNQUE_feature_strred_scale0_score", "FUNQUE_feature_strred_scale1_score", + "FUNQUE_feature_strred_scale2_score", "FUNQUE_feature_strred_scale3_score", - "FUNQUE_feature_ssim_score", + "FUNQUE_feature_ms_ssim_mean_scale0_score", "FUNQUE_feature_ms_ssim_mean_scale1_score", + "FUNQUE_feature_ms_ssim_mean_scale2_score", "FUNQUE_feature_ms_ssim_mean_scale3_score", + "FUNQUE_feature_ms_ssim_cov_scale0_score", "FUNQUE_feature_ms_ssim_cov_scale1_score", + "FUNQUE_feature_ms_ssim_cov_scale2_score", "FUNQUE_feature_ms_ssim_cov_scale3_score", + "FUNQUE_feature_ms_ssim_mink3_scale0_score", "FUNQUE_feature_ms_ssim_mink3_scale1_score", + "FUNQUE_feature_ms_ssim_mink3_scale2_score", "FUNQUE_feature_ms_ssim_mink3_scale3_score", NULL }; diff --git a/libvmaf/src/feature/third_party/funque/funque_adm_options.h b/libvmaf/src/feature/third_party/funque/funque_adm_options.h index 4e76ad44e..aba0bb495 100644 --- a/libvmaf/src/feature/third_party/funque/funque_adm_options.h +++ b/libvmaf/src/feature/third_party/funque/funque_adm_options.h @@ -38,6 +38,6 @@ #define DEFAULT_ADM_LEVELS 4 #define MAX_ADM_LEVELS 4 -#define MIN_ADM_LEVELS 1 +#define MIN_ADM_LEVELS 0 #endif /* ADM_OPTIONS_H_ */ diff --git a/libvmaf/src/feature/third_party/funque/funque_filters.c b/libvmaf/src/feature/third_party/funque/funque_filters.c index 62916674e..c63ae22c8 100644 --- a/libvmaf/src/feature/third_party/funque/funque_filters.c +++ b/libvmaf/src/feature/third_party/funque/funque_filters.c @@ -107,11 +107,10 @@ void funque_vifdwt2_band0(float *src, float *band_a, ptrdiff_t dst_stride, int w aligned_free(tmplo); } -//Convolution using coefficients from python workspace -void spatial_filter(float *src, float *dst, int width, int height) -{ - //Copied the coefficients from python coefficients - float filter_coeffs[21] = { +const float nadeanu_filter_coeffs[5] = {0.0253133196, 0.2310067710, 0.4759767950, 0.2310067710, + 0.0253133196 }; + +const float ngan_filter_coeffs[21] = { -0.01373463642215844680849 , -0.01608514932055564797264 , -0.01890698454168517061991 , @@ -132,58 +131,63 @@ void spatial_filter(float *src, float *dst, int width, int height) -0.02215701978091480159327 , -0.01890698454168517061991 , -0.01608514932055564797264 , - -0.01373463642215844680849 - }; + -0.01373463642215844680849 }; +void spatial_csfs(float *src, float *dst, int width, int height, float *tmp_buf, char *spatial_csf_filter) +{ + const float *filter_coeffs; + int filter_size; + if(strcmp(spatial_csf_filter, "nadenau_spat") == 0) { + /*coefficients for 5 tap nadeanu_spat filter */ + filter_coeffs = nadeanu_filter_coeffs; + filter_size = 5; + } else if (strcmp(spatial_csf_filter, "ngan_spat") == 0) { + filter_coeffs = ngan_filter_coeffs; + filter_size = 21; + } int src_px_stride = width; int dst_px_stride = width; - float *tmp = aligned_malloc(ALIGN_CEIL(src_px_stride * sizeof(float)), MAX_ALIGN); float fcoeff, imgcoeff; int i, j, fi, fj, ii, jj; - int fwidth = 21; - for (i = 0; i < height; ++i) { + for(i = 0; i < height; ++i) { /* Vertical pass. */ - for (j = 0; j < width; ++j) { + for(j = 0; j < width; ++j) { double accum = 0; - for (fi = 0; fi < fwidth; ++fi) { + for(fi = 0; fi < filter_size; ++fi) { fcoeff = filter_coeffs[fi]; - - ii = i - fwidth / 2 + fi; - ii = ii < 0 ? -(ii+1) : (ii >= height ? 2 * height - ii - 1 : ii); + + ii = i - filter_size / 2 + fi; + ii = ii < 0 ? -(ii + 1) : (ii >= height ? 2 * height - ii - 1 : ii); imgcoeff = src[ii * src_px_stride + j]; accum += (double) fcoeff * imgcoeff; } - - tmp[j] = accum; + tmp_buf[j] = accum; } /* Horizontal pass. */ - for (j = 0; j < width; ++j) { + for(j = 0; j < width; ++j) { double accum = 0; - for (fj = 0; fj < fwidth; ++fj) { + for(fj = 0; fj < filter_size; ++fj) { fcoeff = filter_coeffs[fj]; - jj = j - fwidth / 2 + fj; - jj = jj < 0 ? -(jj+1) : (jj >= width ? 2 * width - jj - 1 : jj); + jj = j - filter_size / 2 + fj; + jj = jj < 0 ? -(jj + 1) : (jj >= width ? 2 * width - jj - 1 : jj); - imgcoeff = tmp[jj]; + imgcoeff = tmp_buf[jj]; accum += (double) fcoeff * imgcoeff; } - dst[i * dst_px_stride + j] = accum; } } - aligned_free(tmp); - return; } @@ -199,6 +203,30 @@ void normalize_bitdepth(float *src, float *dst, int scaler, ptrdiff_t dst_stride return; } +void reflect_pad_for_input(const float *src, float *dst, int width, int height, int reflect_width, int reflect_height) +{ + size_t out_width = width + 2 * reflect_width; + size_t out_height = height + 2 * reflect_height; + + for (size_t i = reflect_height; i != (out_height - reflect_height); i++) { + + for (int j = 0; j != reflect_width; j++) + { + dst[i * out_width + (reflect_width - 1 - j)] = src[(i - reflect_height) * width + j + 1]; + } + + memcpy(&dst[i * out_width + reflect_width], &src[(i - reflect_height) * width], sizeof(float) * width); + + for (int j = 0; j != reflect_width; j++) + dst[i * out_width + out_width - reflect_width + j] = dst[i * out_width + out_width - reflect_width - 2 - j]; + } + + for (int i = 0; i != reflect_height; i++) { + memcpy(&dst[(reflect_height - 1) * out_width - i * out_width], &dst[reflect_height * out_width + (i + 1) * out_width], sizeof(float) * out_width); + memcpy(&dst[(out_height - reflect_height) * out_width + i * out_width], &dst[(out_height - reflect_height - 1) * out_width - (i + 1) * out_width], sizeof(float) * out_width); + } +} + void funque_dwt2_inplace_csf(const dwt2buffers *src, float factors[4], int min_theta, int max_theta) { float *src_ptr; diff --git a/libvmaf/src/feature/third_party/funque/funque_filters.h b/libvmaf/src/feature/third_party/funque/funque_filters.h index 6b0cc45cf..4900c4a44 100644 --- a/libvmaf/src/feature/third_party/funque/funque_filters.h +++ b/libvmaf/src/feature/third_party/funque/funque_filters.h @@ -23,6 +23,17 @@ #include #include "macros.h" +// Spatial Filters +#define NGAN_21_TAP_FILTER 21 +#define NADENAU_SPAT_5_TAP_FILTER 5 + +// Wavelet Filters +#define NADENAU_WEIGHT_FILTER 1 // Default set to nadenau_weight +#define LI_FILTER 2 +#define HILL_FILTER 3 +#define WATSON_FILTER 4 +#define MANNOS_WEIGHT_FILTER 5 + struct funque_dwt_model_params { float a; float k; @@ -72,7 +83,49 @@ typedef struct dwt2buffers { ptrdiff_t stride; }dwt2buffers; -void spatial_filter(float *src, float *dst, int width, int height); +/* filter format where 0 = approx, 1 = vertical, 2 = diagonal, 3 = horizontal as in + * funque_dwt2_inplace_csf */ +static const float nadenau_weight_coeffs[4][4] = { + {1, 0.04299846, 0.00556257, 0.04299846}, + {1, 0.42474743, 0.18536903, 0.42474743}, + {1, 0.79592083, 0.63707782, 0.79592083}, + {1, 0.94104990, 0.88686180, 0.94104990}, + /*{ 1, 0.98396102, 0.96855064, 0.98396102},*/ +}; + +static const float li_coeffs[4][4] = { + {1, 0.00544585178, 0.00023055401, 0.00544585178}, + {1, 0.16683506215, 0.04074566701, 0.16683506215}, + {1, 0.66786346496, 0.38921962529, 0.66786346496}, + {1, 0.98626459244, 0.87735995465, 0.98626459244}, + /*{ 1, 0.91608864363, 0.91608864363, 0.98675189575},*/ +}; + +static const float hill_coeffs[4][4] = { + {1, 0.08655904, 0.03830355, 0.08655904}, + {1, -0.33820268, 0.10885236, -0.33820268}, + {1, -0.06836095, -0.20426743, -0.06836095}, + {1, -0.04262307, -0.0619592, -0.04262307}, + /*{ -0.03159788, -0.03394834, -0.03394834, -0.04074054},*/ +}; + +static const float watson_coeffs[4][4] = { + {1, 0.01738153, 0.00589069, 0.01738153}, + {1, 0.03198481, 0.01429907, 0.03198481}, + {1, 0.04337266, 0.02439691, 0.04337266}, + {1, 0.04567341, 0.03131274, 0.04567341}, + /*{ 0.03669688, 0.03867382, 0.03867382, 0.03187392},*/ +}; + +static const float mannos_weight_coeffs[4][4] = { + {1, 7.11828321e-03, 2.43358422e-04, 7.11828321e-03}, + {1, 2.25003123e-01, 5.62679733e-02, 2.25003123e-01}, + {1, 7.82068784e-01, 4.94193706e-01, 7.82068784e-01}, + {1, 9.81000000e-01, 9.81000000e-01, 9.81000000e-01}, + /*{ 9.81000000e-01, 9.81000000e-01, 9.81000000e-01, 9.81000000e-01},*/ +}; + +void spatial_csfs(float *src, float *dst, int width, int height, float *tmp_buf, char *spatial_csf_filter); void funque_dwt2(float *src, dwt2buffers *dwt2_dst, int width, int height); @@ -82,4 +135,6 @@ void normalize_bitdepth(float *src, float *dst, int scaler, ptrdiff_t dst_stride void funque_dwt2_inplace_csf(const dwt2buffers *src, float factors[4], int min_theta, int max_theta); +void reflect_pad_for_input(const float *src, float *dst, int width, int height, int reflect_width, int reflect_height); + #endif /* FILTERS_FUNQUE_H_ */ \ No newline at end of file diff --git a/libvmaf/src/feature/third_party/funque/funque_global_options.h b/libvmaf/src/feature/third_party/funque/funque_global_options.h index c73363119..bd94d8349 100644 --- a/libvmaf/src/feature/third_party/funque/funque_global_options.h +++ b/libvmaf/src/feature/third_party/funque/funque_global_options.h @@ -12,7 +12,7 @@ #define DEFAULT_REF_DISPLAY_HEIGHT (1080) #define MAX_LEVELS 4 -#define MIN_LEVELS 1 +#define MIN_LEVELS 0 #endif //VMAF_FUNQUE_GLOBAL_OPTIONS_H diff --git a/libvmaf/src/feature/third_party/funque/funque_picture_copy.c b/libvmaf/src/feature/third_party/funque/funque_picture_copy.c index 2c280a51a..3e44ba4fb 100644 --- a/libvmaf/src/feature/third_party/funque/funque_picture_copy.c +++ b/libvmaf/src/feature/third_party/funque/funque_picture_copy.c @@ -23,13 +23,13 @@ #include "funque_filters.h" void funque_picture_copy_hbd(float *dst, ptrdiff_t dst_stride, - VmafPicture *src, int offset) + VmafPicture *src, int offset, int width, int height) { float *float_data = dst; uint16_t *data = src->data[0]; - for (unsigned i = 0; i < src->h[0]; i++) { - for (unsigned j = 0; j < src->w[0]; j++) { + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { float_data[j] = (float) data[j] + offset; } float_data += dst_stride / sizeof(float); @@ -39,16 +39,16 @@ void funque_picture_copy_hbd(float *dst, ptrdiff_t dst_stride, } void funque_picture_copy(float *dst, ptrdiff_t dst_stride, - VmafPicture *src, int offset, unsigned bpc) + VmafPicture *src, int offset, unsigned bpc, int width, int height) { if (bpc > 8) - return funque_picture_copy_hbd(dst, dst_stride, src, offset); + return funque_picture_copy_hbd(dst, dst_stride, src, offset, width, height); float *float_data = dst; uint8_t *data = src->data[0]; - for (unsigned i = 0; i < src->h[0]; i++) { - for (unsigned j = 0; j < src->w[0]; j++) { + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { float_data[j] = (float) data[j] + offset; } float_data += dst_stride / sizeof(float); diff --git a/libvmaf/src/feature/third_party/funque/funque_picture_copy.h b/libvmaf/src/feature/third_party/funque/funque_picture_copy.h index b4017732d..4eb0d54dd 100644 --- a/libvmaf/src/feature/third_party/funque/funque_picture_copy.h +++ b/libvmaf/src/feature/third_party/funque/funque_picture_copy.h @@ -18,4 +18,4 @@ #include void funque_picture_copy(float *dst, ptrdiff_t dst_stride, VmafPicture *src, - int offset, unsigned bpc); + int offset, unsigned bpc, int width, int height); diff --git a/libvmaf/src/feature/third_party/funque/funque_ssim.c b/libvmaf/src/feature/third_party/funque/funque_ssim.c index 3e1d743ba..7077fa84d 100644 --- a/libvmaf/src/feature/third_party/funque/funque_ssim.c +++ b/libvmaf/src/feature/third_party/funque/funque_ssim.c @@ -21,6 +21,9 @@ #include #include "funque_filters.h" #include "funque_ssim_options.h" +#include +#include +#include int compute_ssim_funque(dwt2buffers *ref, dwt2buffers *dist, double *score, int max_val, float K1, float K2) { @@ -93,5 +96,217 @@ int compute_ssim_funque(dwt2buffers *ref, dwt2buffers *dist, double *score, int ret = 0; + return ret; +} + +const double exps[5] = {0.0448000000, 0.2856000000, 0.3001000000, 0.2363000000, 0.1333000000}; + +int compute_ms_ssim_funque(dwt2buffers* ref, dwt2buffers* dist, MsSsimScore* score, int max_val, + float K1, float K2, int n_levels) +{ + int ret = 1; + + int cum_array_width = (ref->width) * (1 << n_levels); + + int width = ref->width; + int height = ref->height; + + float C1 = (K1 * max_val) * (K1 * max_val); + float C2 = (K2 * max_val) * (K2 * max_val); + + float var_x = 0; + float var_y = 0; + float cov_xy = 0; + + float* var_x_cum = *(score->var_x_cum); + float* var_y_cum = *(score->var_y_cum); + float* cov_xy_cum = *(score->cov_xy_cum); + + + double cube_1minus_l = 0; + double cube_1minus_cs = 0; + double cube_1minus_map = 0; + + + int win_dim = (1 << n_levels); // 2^L + int win_size = (1 << (n_levels << 1)); // 2^(2L), i.e., a win_dim X win_dim square + + double mx, my, l, cs; + double ssim_sum = 0; + double l_sum = 0; + double cs_sum = 0; + double ssim_sq_sum = 0; + double l_sq_sum = 0; + double cs_sq_sum = 0; + int index = 0; + int index_cum = 0; + for(int i = 0; i < height; i++) { + for(int j = 0; j < width; j++) { + index = i * width + j; + mx = ref->bands[0][index] / win_dim; + my = dist->bands[0][index] / win_dim; + + for(int k = 1; k < 4; k++) { + var_x_cum[index_cum] += ((ref->bands[k][index] * ref->bands[k][index]) / win_size); + var_y_cum[index_cum] += ((dist->bands[k][index] * dist->bands[k][index]) / win_size); + cov_xy_cum[index_cum] += ((ref->bands[k][index] * dist->bands[k][index]) / win_size); + } + + var_x = var_x_cum[index_cum]; + var_y = var_y_cum[index_cum]; + cov_xy = cov_xy_cum[index_cum]; + + l = (2 * mx * my + C1) / ((mx * mx) + (my * my) + C1); + cs = (2 * cov_xy + C2) / (var_x + var_y + C2); + + cube_1minus_l += pow((1 - l), 3); + cube_1minus_cs += pow((1 - cs), 3); + cube_1minus_map += pow((1 - (l * cs)), 3); + + ssim_sum += (l * cs); + l_sum += l; + cs_sum += cs; + ssim_sq_sum += (l * cs) * (l * cs); + l_sq_sum += l * l; + cs_sq_sum += cs * cs; + + index_cum++; + } + index_cum += (cum_array_width - width); + } + + + double l_mink3 = 1 - (double)cbrt(cube_1minus_l / (width * height)); + double cs_mink3 = 1 - (double)cbrt(cube_1minus_cs / (width * height)); + double ssim_mink3 = 1 - (double)cbrt(cube_1minus_map / (width * height)); + score->ssim_mink3 = ssim_mink3; + score->l_mink3 = l_mink3; + score->cs_mink3 = cs_mink3; + //score->ssim_mean = ssim_clip(ssim_val, 0, 1); + + double ssim_mean = ssim_sum / (height * width); + double l_mean = l_sum / (height * width); + double cs_mean = cs_sum / (height * width); + score->ssim_mean = ssim_mean; + score->l_mean = l_mean; + score->cs_mean = cs_mean; + + double l_var = (l_sq_sum / (height * width)) - (l_mean * l_mean); + double cs_var = (cs_sq_sum / (height * width)) - (cs_mean * cs_mean); + double ssim_var = (ssim_sq_sum / (height * width)) - (ssim_mean * ssim_mean); + + assert(l_var >= 0); + assert(cs_var >= 0); + assert(ssim_var >= 0); + + double l_std = sqrt(l_var); + double cs_std = sqrt(cs_var); + double ssim_std = sqrt(ssim_var); + + double l_cov = l_std / l_mean; + double cs_cov = cs_std / cs_mean; + double ssim_cov = ssim_std / ssim_mean; + + score->ssim_cov = ssim_cov; + score->l_cov = l_cov; + score->cs_cov = cs_cov; + + + ret = 0; + + return ret; +} + +int mean_2x2_ms_ssim_funque(float *var_x_cum, float *var_y_cum, float *cov_xy_cum, + int width, int height, int level) +{ + int ret = 1; + + int index = 0; + int index_cum = 0; + int cum_array_width = (width) * (1 << (level + 1)); + + for(int i = 0; i < (height / 2); i++) { + for(int j = 0; j < (width / 2); j++) { + /* Accumulations are done using mean computations of 2x2 pixels */ + index = i * cum_array_width + j; + var_x_cum[index] = var_x_cum[index_cum] + var_x_cum[index_cum + 1] + + var_x_cum[index_cum + (cum_array_width)] + + var_x_cum[index_cum + (cum_array_width) + 1]; + var_x_cum[index] = var_x_cum[index] / 4; + + var_y_cum[index] = var_y_cum[index_cum] + var_y_cum[index_cum + 1] + + var_y_cum[index_cum + (cum_array_width)] + + var_y_cum[index_cum + (cum_array_width) + 1]; + var_y_cum[index] = var_y_cum[index] / 4; + + cov_xy_cum[index] = cov_xy_cum[index_cum] + cov_xy_cum[index_cum + 1] + + cov_xy_cum[index_cum + (cum_array_width)] + + cov_xy_cum[index_cum + (cum_array_width) + 1]; + cov_xy_cum[index] = cov_xy_cum[index] / 4; + + index_cum += 2; + } + index_cum += ((cum_array_width * 2) - width); + } + + ret = 0; + + return ret; +} + +int compute_ms_ssim_mean_scales(MsSsimScore* score, int n_levels) +{ + int ret = 1; + + double cum_prod_mean[5] = {0}; + double cum_prod_concat_mean[5] = {0}; + + double cum_prod_cov[5] = {0}; + double cum_prod_concat_cov[5] = {0}; + + double cum_prod_mink3[5] = {0}; + double cum_prod_concat_mink3[5] = {0}; + + double sign_cum_prod_mean = (score[0].cs_mean) >= 0 ? 1 : -1; + double sign_cum_prod_cov = (score[0].cs_cov) >= 0 ? 1 : -1; + double sign_cum_prod_mink3 = (score[0].cs_mink3) >= 0 ? 1 : -1; + + cum_prod_mean[0] = pow(fabs(score[0].cs_mean), exps[0]) * sign_cum_prod_mean; + cum_prod_cov[0] = pow(fabs(score[0].cs_cov), exps[0]) * sign_cum_prod_cov; + cum_prod_mink3[0] = pow(fabs(score[0].cs_mink3), exps[0]) * sign_cum_prod_mink3; + + for(int i = 1; i < n_levels; i++) { + sign_cum_prod_mean = (score[i].cs_mean) >= 0 ? 1 : -1; + sign_cum_prod_cov = (score[i].cs_cov) >= 0 ? 1 : -1; + sign_cum_prod_mink3 = (score[i].cs_mink3) >= 0 ? 1 : -1; + + cum_prod_mean[i] = cum_prod_mean[i-1] * pow(fabs(score[i].cs_mean), exps[i]) * sign_cum_prod_mean; + cum_prod_cov[i] = cum_prod_cov[i - 1] * pow(fabs(score[i].cs_cov), exps[i]) * sign_cum_prod_cov; + cum_prod_mink3[i] = cum_prod_mink3[i - 1] * pow(fabs(score[i].cs_mink3), exps[i]) * sign_cum_prod_mink3; + } + + cum_prod_concat_mean[0] = 1; + cum_prod_concat_cov[0] = 1; + cum_prod_concat_mink3[0] = 1; + for(int i = 1; i < n_levels; i++) { + cum_prod_concat_mean[i] = cum_prod_mean[i - 1]; + cum_prod_concat_cov[i] = cum_prod_cov[i - 1]; + cum_prod_concat_mink3[i] = cum_prod_mink3[i - 1]; + } + + for(int i = 0; i < n_levels; i++) { + + float sign_mssim_mean = (score[i].ssim_mean) >= 0 ? 1 : -1; + float sign_mssim_cov = (score[i].ssim_cov) >= 0 ? 1 : -1; + float sign_mssim_mink3 = (score[i].ssim_mink3) >= 0 ? 1 : -1; + score[i].ms_ssim_mean = cum_prod_concat_mean[i] * pow(fabs(score[i].ssim_mean), exps[i]) * sign_mssim_mean; + score[i].ms_ssim_cov = cum_prod_concat_cov[i] * pow(fabs(score[i].ssim_cov), exps[i])* sign_mssim_cov; + score[i].ms_ssim_mink3 = cum_prod_concat_mink3[i] * pow(fabs(score[i].ssim_mink3), exps[i])* sign_mssim_mink3; + + } + + ret = 0; + return ret; } \ No newline at end of file diff --git a/libvmaf/src/feature/third_party/funque/funque_ssim.h b/libvmaf/src/feature/third_party/funque/funque_ssim.h index bc53ebf58..afcea0e2b 100644 --- a/libvmaf/src/feature/third_party/funque/funque_ssim.h +++ b/libvmaf/src/feature/third_party/funque/funque_ssim.h @@ -16,4 +16,8 @@ * */ -int compute_ssim_funque(dwt2buffers *ref, dwt2buffers *dist, double *score, int max_val, float K1, float K2); \ No newline at end of file +int compute_ssim_funque(dwt2buffers *ref, dwt2buffers *dist, double *score, int max_val, float K1, float K2); +int compute_ms_ssim_funque(dwt2buffers *ref, dwt2buffers *dist, MsSsimScore *score, int max_val, + float K1, float K2, int n_levels); +int mean_2x2_ms_ssim_funque(float *var_x_cum, float *var_y_cum, float *cov_xy_cum, int width, int height, int level); +int compute_ms_ssim_mean_scales(MsSsimScore* score, int n_levels); \ No newline at end of file diff --git a/libvmaf/src/feature/third_party/funque/funque_ssim_options.h b/libvmaf/src/feature/third_party/funque/funque_ssim_options.h index 0a2bc6b8d..7cdcc86d6 100644 --- a/libvmaf/src/feature/third_party/funque/funque_ssim_options.h +++ b/libvmaf/src/feature/third_party/funque/funque_ssim_options.h @@ -19,9 +19,31 @@ #ifndef FUNQUE_SSIM_OPTIONS_H_ #define FUNQUE_SSIM_OPTIONS_H_ -#define ENABLE_MINK3POOL 1 +#define ENABLE_MINK3POOL 0 +#define ENABLE_PADDING 0 #define DEFAULT_SSIM_LEVELS 4 +#define DEFAULT_MS_SSIM_LEVELS 4 + +typedef struct MsSsimScore { + double ssim_mean; + double l_mean; + double cs_mean; + double ssim_cov; + double l_cov; + double cs_cov; + double ssim_mink3; + double l_mink3; + double cs_mink3; + + double ms_ssim_mean; + double ms_ssim_cov; + double ms_ssim_mink3; + + float **var_x_cum; + float **var_y_cum; + float **cov_xy_cum; +} MsSsimScore; static inline double ssim_clip(double value, double low, double high) { diff --git a/libvmaf/src/feature/third_party/funque/funque_strred.c b/libvmaf/src/feature/third_party/funque/funque_strred.c new file mode 100644 index 000000000..1447ad297 --- /dev/null +++ b/libvmaf/src/feature/third_party/funque/funque_strred.c @@ -0,0 +1,311 @@ +/** + * + * Copyright 2016-2020 Netflix, Inc. + * + * Licensed under the BSD+Patent License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSDplusPatent + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include + +#include "funque_filters.h" +#include "funque_strred_options.h" +#include "funque_strred.h" + +void strred_reflect_pad(const float* src, size_t width, size_t height, int reflect, float* dest) +{ + size_t out_width = width + 2 * reflect; + size_t out_height = height + 2 * reflect; + + for(size_t i = reflect; i != (out_height - reflect); i++) { + for(int j = 0; j != reflect; j++) { + dest[i * out_width + (reflect - 1 - j)] = src[(i - reflect) * width + j + 1]; + } + + memcpy(&dest[i * out_width + reflect], &src[(i - reflect) * width], sizeof(float) * width); + + for(int j = 0; j != reflect; j++) + dest[i * out_width + out_width - reflect + j] = + dest[i * out_width + out_width - reflect - 2 - j]; + } + + for(int i = 0; i != reflect; i++) { + memcpy(&dest[(reflect - 1) * out_width - i * out_width], + &dest[reflect * out_width + (i + 1) * out_width], sizeof(float) * out_width); + memcpy(&dest[(out_height - reflect) * out_width + i * out_width], + &dest[(out_height - reflect - 1) * out_width - (i + 1) * out_width], + sizeof(float) * out_width); + } +} + +void strred_strred_integral_image_2(const float* src1, const float* src2, size_t width, + size_t height, double* sum) +{ + for(size_t i = 0; i < (height + 1); ++i) { + for(size_t j = 0; j < (width + 1); ++j) { + if(i == 0 || j == 0) + continue; + + double val = + (double) src1[(i - 1) * width + (j - 1)] * (double) src2[(i - 1) * width + (j - 1)]; + + if(i >= 1) { + val += sum[(i - 1) * (width + 1) + j]; + if(j >= 1) { + val += sum[i * (width + 1) + j - 1] - sum[(i - 1) * (width + 1) + j - 1]; + } + } else { + if(j >= 1) { + val += sum[i * width + j - 1]; + } + } + sum[i * (width + 1) + j] = val; + } + } +} + +void strred_integral_image(const float* src, size_t width, size_t height, double* sum) +{ + for(size_t i = 0; i < (height + 1); ++i) { + for(size_t j = 0; j < (width + 1); ++j) { + if(i == 0 || j == 0) + continue; + + double val = (double) src[(i - 1) * width + (j - 1)]; + + if(i >= 1) { + val += sum[(i - 1) * (width + 1) + j]; + if(j >= 1) { + val += sum[i * (width + 1) + j - 1] - sum[(i - 1) * (width + 1) + j - 1]; + } + } else { + if(j >= 1) { + val += sum[i * width + j - 1]; + } + } + sum[i * (width + 1) + j] = val; + } + } +} + +void strred_compute_entropy_scale(const double* int_1_x, const double* int_2_x, size_t width, + size_t height, size_t kh, size_t kw, int kNorm, float* entropy, + float* scale) +{ + float mu_x, var_x; + float entr_const = log(2 * PI_CONSTANT * EULERS_CONSTANT); + float sigma_nsq = STRRED_SIGMA_NSQ; + + for(size_t i = 0; i < (height - kh); i++) { + for(size_t j = 0; j < (width - kw); j++) { + mu_x = (int_1_x[i * width + j] - int_1_x[i * width + j + kw] - + int_1_x[(i + kh) * width + j] + int_1_x[(i + kh) * width + j + kw]) / + kNorm; + var_x = ((int_2_x[i * width + j] - int_2_x[i * width + j + kw] - + int_2_x[(i + kh) * width + j] + int_2_x[(i + kh) * width + j + kw]) / + kNorm) - (mu_x * mu_x); + + var_x = (var_x < 0) ? 0 : var_x; /* Add CLIP macro */ + + entropy[i * width + j] = log(var_x + sigma_nsq) + entr_const; + scale[i * width + j] = log(1 + var_x); + } + } +} + +void rred_entropies_and_scales(const float* src, int block_size, size_t width, size_t height, + float* entropy, float* scale) +{ + if(block_size == 1) { + int k = STRRED_WINDOW_SIZE; + int k_norm = k * k; + int kw = k; + int kh = k; + + int x_reflect = (int) ((kw - 1) / 2); + size_t r_width = width + (2 * x_reflect); + size_t r_height = height + (2 * x_reflect); + + float* x_pad = + (float*) malloc(sizeof(float) * (width + (2 * x_reflect)) * (height + (2 * x_reflect))); + double* int_1_x = (double*) calloc((r_width + 1) * (r_height + 1), sizeof(double)); + double* int_2_x = (double*) calloc((r_width + 1) * (r_height + 1), sizeof(double)); + + strred_reflect_pad(src, width, height, x_reflect, x_pad); + strred_integral_image(x_pad, r_width, r_height, int_1_x); + strred_strred_integral_image_2(x_pad, x_pad, r_width, r_height, int_2_x); + strred_compute_entropy_scale(int_1_x, int_2_x, r_width + 1, r_height + 1, kw, kh, k_norm, + entropy, scale); + + free(x_pad); + free(int_1_x); + free(int_2_x); + } +} + +void subract_subbands(const float* ref_src, const float* ref_prev_src, float* ref_dst, + const float* dist_src, const float* dist_prev_src, float* dist_dst, + size_t width, size_t height) +{ + size_t i, j; + + for(i = 0; i < height; i++) { + for(j = 0; j < width; j++) { + ref_dst[i * width + j] = ref_src[i * width + j] - ref_prev_src[i * width + j]; + dist_dst[i * width + j] = dist_src[i * width + j] - dist_prev_src[i * width + j]; + } + } +} + +int copy_prev_frame_strred_funque(const struct dwt2buffers* ref, const struct dwt2buffers* dist, + struct strredbuffers* prev_ref, struct strredbuffers* prev_dist, + size_t width, size_t height) +{ + int subband; + int total_subbands = DEFAULT_STRRED_SUBBANDS; + + for(subband = 1; subband < total_subbands; subband++) { + memcpy(prev_ref->bands[subband], ref->bands[subband], width * height * sizeof(float)); + memcpy(prev_dist->bands[subband], dist->bands[subband], width * height * sizeof(float)); + } + prev_ref->width = ref->width; + prev_ref->height = ref->height; + prev_ref->stride = ref->stride; + + prev_dist->width = dist->width; + prev_dist->height = dist->height; + prev_dist->stride = dist->stride; + + return 0; +} + +int compute_strred_funque(const struct dwt2buffers* ref, const struct dwt2buffers* dist, + struct strredbuffers* prev_ref, struct strredbuffers* prev_dist, + size_t width, size_t height, struct strred_results* strred_scores, + int block_size, int level) +{ + size_t subband; + float spat_abs, spat_values[DEFAULT_STRRED_SUBBANDS]; + float temp_abs, temp_values[DEFAULT_STRRED_SUBBANDS]; + + size_t total_subbands = DEFAULT_STRRED_SUBBANDS; + size_t x_reflect = (size_t) ((STRRED_WINDOW_SIZE - 1) / 2); + size_t r_width = width + (2 * x_reflect); + size_t r_height = height + (2 * x_reflect); + + float* entropies_ref = (float*) calloc((r_width + 1) * (r_height + 1), sizeof(float)); + float* entropies_dist = (float*) calloc((r_width + 1) * (r_height + 1), sizeof(float)); + float* spat_scales_ref = (float*) calloc((r_width + 1) * (r_height + 1), sizeof(float)); + float* spat_scales_dist = (float*) calloc((r_width + 1) * (r_height + 1), sizeof(float)); + float* temp_scales_ref = (float*) calloc((r_width + 1) * (r_height + 1), sizeof(float)); + float* temp_scales_dist = (float*) calloc((r_width + 1) * (r_height + 1), sizeof(float)); + float* spat_aggregate = (float*) calloc((r_width + 1) * (r_height + 1), sizeof(float)); + + for(subband = 1; subband < total_subbands; subband++) { + size_t i, j; + spat_abs = 0; + + rred_entropies_and_scales(ref->bands[subband], block_size, width, height, entropies_ref, + spat_scales_ref); + rred_entropies_and_scales(dist->bands[subband], block_size, width, height, entropies_dist, + spat_scales_dist); + + for(i = 0; i < r_height; i++) { + for(j = 0; j < r_width; j++) { + spat_aggregate[i * r_width + j] = + entropies_ref[i * r_width + j] * spat_scales_ref[i * r_width + j] - + entropies_dist[i * r_width + j] * spat_scales_dist[i * r_width + j]; + } + } + + for(i = 0; i < r_height; i++) { + for(j = 0; j < r_width; j++) { + spat_abs += fabs(spat_aggregate[i * r_width + j]); + } + } + spat_values[subband] = spat_abs / (height * width); + + if(prev_ref != NULL && prev_dist != NULL) { + float* ref_temporal = (float*) calloc((width) * (height), sizeof(float)); + float* dist_temporal = (float*) calloc((width) * (height), sizeof(float)); + float* temp_aggregate = (float*) calloc((r_width + 1) * (r_height + 1), sizeof(float)); + + temp_abs = 0; + + subract_subbands(ref->bands[subband], prev_ref->bands[subband], ref_temporal, + dist->bands[subband], prev_dist->bands[subband], dist_temporal, width, + height); + + rred_entropies_and_scales(ref_temporal, block_size, width, height, entropies_ref, + temp_scales_ref); + rred_entropies_and_scales(dist_temporal, block_size, width, height, entropies_dist, + temp_scales_dist); + + for(i = 0; i < r_height; i++) { + for(j = 0; j < r_width; j++) { + temp_aggregate[i * r_width + j] = + entropies_ref[i * r_width + j] * spat_scales_ref[i * r_width + j] * + temp_scales_ref[i * r_width + j] - + entropies_dist[i * r_width + j] * spat_scales_dist[i * r_width + j] * + temp_scales_dist[i * r_width + j]; + } + } + + for(i = 0; i < r_height; i++) { + for(j = 0; j < r_width; j++) { + temp_abs += fabs(temp_aggregate[i * r_width + j]); + } + } + temp_values[subband] = temp_abs / (height * width); + + free(ref_temporal); + free(dist_temporal); + free(temp_aggregate); + } else { + strred_scores->temp_vals[level] = 0; + strred_scores->spat_temp_vals[level] = 0; + } + } + + strred_scores->spat_vals[level] = (spat_values[1] + spat_values[2] + spat_values[3]) / 3; + strred_scores->temp_vals[level] = (temp_values[1] + temp_values[2] + temp_values[3]) / 3; + strred_scores->spat_temp_vals[level] = + strred_scores->spat_vals[level] * strred_scores->temp_vals[level]; + + // Add equations to compute ST-RRED using norm factors + int norm_factor, num_level; + for(num_level = 0; num_level <= level; num_level++) + norm_factor = num_level + 1; + + strred_scores->spat_vals_cumsum += strred_scores->spat_vals[level]; + strred_scores->temp_vals_cumsum += strred_scores->temp_vals[level]; + strred_scores->spat_temp_vals_cumsum += strred_scores->spat_temp_vals[level]; + + strred_scores->srred_vals[level] = strred_scores->spat_vals_cumsum / norm_factor; + strred_scores->trred_vals[level] = strred_scores->temp_vals_cumsum / norm_factor; + strred_scores->strred_vals[level] = strred_scores->spat_temp_vals_cumsum / norm_factor; + + free(entropies_ref); + free(entropies_dist); + free(spat_scales_ref); + free(spat_scales_dist); + free(temp_scales_ref); + free(temp_scales_dist); + free(spat_aggregate); + + return 0; +} \ No newline at end of file diff --git a/libvmaf/src/feature/third_party/funque/funque_strred.h b/libvmaf/src/feature/third_party/funque/funque_strred.h new file mode 100644 index 000000000..0f1d7c6bf --- /dev/null +++ b/libvmaf/src/feature/third_party/funque/funque_strred.h @@ -0,0 +1,49 @@ +/** + * + * Copyright 2016-2020 Netflix, Inc. + * + * Licensed under the BSD+Patent License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSDplusPatent + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "funque_global_options.h" +#include "funque_strred_options.h" + +typedef struct strredbuffers { + float* bands[4]; + int width; + int height; + int crop_width; + int crop_height; + ptrdiff_t stride; +} strredbuffers; + +typedef struct strred_results { + double srred_vals[MAX_LEVELS]; + double trred_vals[MAX_LEVELS]; + double strred_vals[MAX_LEVELS]; + double spat_vals[MAX_LEVELS]; + double temp_vals[MAX_LEVELS]; + double spat_temp_vals[MAX_LEVELS]; + double spat_vals_cumsum, temp_vals_cumsum, spat_temp_vals_cumsum; + +} strred_results; + +int compute_strred_funque(const struct dwt2buffers* ref, const struct dwt2buffers* dist, + struct strredbuffers* prev_ref, struct strredbuffers* prev_dist, + size_t width, size_t height, struct strred_results* strred_scores, + int block_size, int level); + +int copy_prev_frame_strred_funque(const struct dwt2buffers* ref, const struct dwt2buffers* dist, + struct strredbuffers* prev_ref, struct strredbuffers* prev_dist, + size_t width, size_t height); \ No newline at end of file diff --git a/libvmaf/src/feature/third_party/funque/funque_strred_options.h b/libvmaf/src/feature/third_party/funque/funque_strred_options.h new file mode 100644 index 000000000..2e8d60bf3 --- /dev/null +++ b/libvmaf/src/feature/third_party/funque/funque_strred_options.h @@ -0,0 +1,35 @@ +/** + * + * Copyright 2016-2020 Netflix, Inc. + * + * Licensed under the BSD+Patent License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSDplusPatent + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#ifndef STRRED_OPTIONS_H_ +#define STRRED_OPTIONS_H_ + +#define STRRED_SIGMA_NSQ 0.1 +#define STRRED_WINDOW_SIZE 9 + +#define PI_CONSTANT 3.1415926535897932384626433832795028841971693993751 +#define EULERS_CONSTANT 2.71828182845904523536028747135266249775724709369995 + +#define BLOCK_SIZE 1 + +#define DEFAULT_STRRED_LEVELS 4 +#define DEFAULT_STRRED_SUBBANDS 4 + +#endif /* STRRED_OPTIONS_H_ */ diff --git a/libvmaf/src/feature/third_party/funque/resizer.h b/libvmaf/src/feature/third_party/funque/resizer.h index aaedc25a4..9fa8b3ddc 100644 --- a/libvmaf/src/feature/third_party/funque/resizer.h +++ b/libvmaf/src/feature/third_party/funque/resizer.h @@ -32,6 +32,7 @@ : X) #define MAX(LEFT, RIGHT) (LEFT > RIGHT ? LEFT : RIGHT) #define MIN(LEFT, RIGHT) (LEFT < RIGHT ? LEFT : RIGHT) +#define MAX5(A, B, C, D, E) MAX(MAX(MAX(MAX(A, B), C), D), E) typedef struct ResizerState { diff --git a/libvmaf/src/meson.build b/libvmaf/src/meson.build index 39f1361b4..5660eaa91 100644 --- a/libvmaf/src/meson.build +++ b/libvmaf/src/meson.build @@ -508,7 +508,8 @@ if funque_float_enabled funque_feature_dir + 'funque_adm.c', funque_feature_dir + 'funque_ssim.c', funque_feature_dir + 'resizer.c', - funque_feature_dir + 'hbd_resizer.c' + funque_feature_dir + 'hbd_resizer.c', + funque_feature_dir + 'funque_strred.c' ] endif diff --git a/libvmaf/src/output.c b/libvmaf/src/output.c index 0965939d8..a70123cc1 100644 --- a/libvmaf/src/output.c +++ b/libvmaf/src/output.c @@ -99,7 +99,7 @@ int vmaf_write_output_xml(VmafContext *vmaf, VmafFeatureCollector *fc, int err = vmaf_feature_score_pooled(vmaf, feature_name, j, &score, 0, pic_cnt - 1); if (!err) - fprintf(outfile, "%s=\"%.6f\" ", pool_method_name[j], score); + fprintf(outfile, "%s=\"%.30f\" ", pool_method_name[j], score); } fprintf(outfile, "/>\n"); } @@ -167,7 +167,7 @@ int vmaf_write_output_json(VmafContext *vmaf, VmafFeatureCollector *fc, case FP_NORMAL: case FP_ZERO: case FP_SUBNORMAL: - fprintf(outfile, " \"%s\": %.6f%s\n", + fprintf(outfile, " \"%s\": %.30f%s\n", vmaf_feature_name_alias(fc->feature_vector[j]->name), fc->feature_vector[j]->score[i].value, cnt2 < cnt ? "," : "" @@ -203,7 +203,7 @@ int vmaf_write_output_json(VmafContext *vmaf, VmafFeatureCollector *fc, case FP_NORMAL: case FP_ZERO: case FP_SUBNORMAL: - fprintf(outfile, " \"%s\": %.6f", + fprintf(outfile, " \"%s\": %.30f", pool_method_name[j], score); break; case FP_INFINITE: @@ -273,7 +273,7 @@ int vmaf_write_output_csv(VmafFeatureCollector *fc, FILE *outfile, continue; if (!fc->feature_vector[j]->score[i].written) continue; - fprintf(outfile, "%.6f,", fc->feature_vector[j]->score[i].value); + fprintf(outfile, "%.30f,", fc->feature_vector[j]->score[i].value); } fprintf(outfile, "\n"); } @@ -303,7 +303,7 @@ int vmaf_write_output_sub(VmafFeatureCollector *fc, FILE *outfile, continue; if (!fc->feature_vector[j]->score[i].written) continue; - fprintf(outfile, "%s: %.6f|", + fprintf(outfile, "%s: %.30f|", vmaf_feature_name_alias(fc->feature_vector[j]->name), fc->feature_vector[j]->score[i].value); } diff --git a/model/funque_float.json b/model/funque_float.json index f7f8b9ec0..98e834def 100644 --- a/model/funque_float.json +++ b/model/funque_float.json @@ -13,20 +13,39 @@ "model_type": "LIBSVMNUSVR", "feature_opts_dicts": [ { + "enable_spatial_csf": false, "enable_resize": true, - "vif_levels": 2 - }, - {}, - {}, - {}, - {} + + "spatial_csf_filter": 5, + "wavelet_csf_filter": 1, + + "vif_levels": 4, + "adm_levels": 4, + "ssim_levels": 4, + "ms_ssim_levels": 4, + "strred_levels": 4 + } ], "feature_names": [ - "FUNQUE_feature_vif_scale0_score", - "FUNQUE_feature_vif_scale1_score", - "FUNQUE_feature_motion_score", - "FUNQUE_feature_adm2_score", - "FUNQUE_float_ssim" + "FUNQUE_feature_vif_score", + "FUNQUE_feature_vif_scale0_score", "FUNQUE_feature_vif_scale1_score", + "FUNQUE_feature_vif_scale2_score", "FUNQUE_feature_vif_scale3_score", + + "FUNQUE_feature_adm_score", + "FUNQUE_feature_adm_scale0_score", "FUNQUE_feature_adm_scale1_score", + "FUNQUE_feature_adm_scale2_score", "FUNQUE_feature_adm_scale3_score", + + "FUNQUE_feature_ssim_scale0_score", "FUNQUE_feature_ssim_scale1_score", + "FUNQUE_feature_ssim_scale2_score", "FUNQUE_feature_ssim_scale3_score", + + "FUNQUE_feature_strred_scale0_score", "FUNQUE_feature_strred_scale1_score", + "FUNQUE_feature_strred_scale2_score", "FUNQUE_feature_strred_scale3_score", + "FUNQUE_feature_ms_ssim_mean_scale0_score", "FUNQUE_feature_ms_ssim_mean_scale1_score", + "FUNQUE_feature_ms_ssim_mean_scale2_score", "FUNQUE_feature_ms_ssim_mean_scale3_score", + "FUNQUE_feature_ms_ssim_cov_scale0_score", "FUNQUE_feature_ms_ssim_cov_scale1_score", + "FUNQUE_feature_ms_ssim_cov_scale2_score", "FUNQUE_feature_ms_ssim_cov_scale3_score", + "FUNQUE_feature_ms_ssim_mink3_scale0_score", "FUNQUE_feature_ms_ssim_mink3_scale1_score", + "FUNQUE_feature_ms_ssim_mink3_scale2_score", "FUNQUE_feature_ms_ssim_mink3_scale3_score" ], "slopes": [ 0.013377926421404682, @@ -48,11 +67,24 @@ "model": "svm_type nu_svr\nkernel_type rbf\ngamma 0.05\nnr_class 2\ntotal_sv 84\nrho 0.0927242\nSV\n4 1:0.83936445 2:0.065893164 3:0.20281726 4:0.85020825 5:0.8786392 \n-4 1:0.88399057 2:0.065893164 3:0.13911393 4:0.89299774 5:0.91389091 \n-4 1:0.90970067 2:0.065893164 3:0.10332676 4:0.91761998 5:0.93350919 \n-4 1:0.93721974 2:0.065893164 3:0.066889505 4:0.93914732 5:0.95039068 \n-4 1:0.96080091 2:0.065893164 3:0.038791505 4:0.95561133 5:0.96328362 \n4 1:0.85683245 2:0.065893164 3:0.21958222 4:0.84345413 5:0.86942086 \n-1.943388131866539 1:0.91845639 2:0.065893164 3:0.10110662 4:0.93272892 5:0.94314437 \n4 1:0.94440382 2:0.065893164 3:0.066034206 4:0.95713921 5:0.96295097 \n-4 1:0.96673844 2:0.065893164 3:0.038520646 4:0.9725482 5:0.97493599 \n4 1:0.74739764 2:0.14846153 3:0.41113023 4:0.77153239 5:0.83007162 \n4 1:0.83427608 2:0.14846153 3:0.29005041 4:0.84614677 5:0.88820128 \n4 1:0.88770853 2:0.14846153 3:0.19790088 4:0.89398286 5:0.92365587 \n-4 1:0.92771426 2:0.14846153 3:0.12381822 4:0.92637858 5:0.94684099 \n-4 1:0.9676263 2:0.14846153 3:0.052060283 4:0.95559205 5:0.96703772 \n4 1:0.78152183 2:0.14846153 3:0.40864318 4:0.82425745 5:0.86490547 \n4 1:0.87081852 2:0.14846153 3:0.26806015 4:0.91075502 5:0.93034494 \n-4 1:0.95651903 2:0.14846153 3:0.1225211 4:0.9517698 5:0.96094449 \n-4 1:0.98981768 2:0.14846153 3:0.05742821 4:0.97422671 5:0.97760939 \n4 1:0.86791324 2:1.3877788e-17 3:0.37260704 4:0.87857405 5:0.91034337 \n4 1:0.92130668 2:1.3877788e-17 3:0.24261788 4:0.92438751 5:0.94390258 \n-4 1:0.95893545 2:1.3877788e-17 3:0.14600092 4:0.95136652 5:0.96312595 \n-4 1:0.98279931 2:1.3877788e-17 3:0.088715839 4:0.96562426 5:0.97317865 \n-4 1:0.99598932 2:1.3877788e-17 3:0.054338915 4:0.97258446 5:0.97789853 \n4 1:0.88979042 2:1.3877788e-17 3:0.36458565 4:0.91829942 5:0.93438331 \n4 1:0.92682983 2:1.3877788e-17 3:0.29325466 4:0.92327294 5:0.94029189 \n4 1:0.96549689 2:1.3877788e-17 3:0.18448706 4:0.95568435 5:0.96462325 \n-4 1:0.98816752 2:1.3877788e-17 3:0.11216929 4:0.97208429 5:0.97716506 \n-4 1:1 2:1.3877788e-17 3:0.074709209 4:0.97798793 5:0.98145195 \n-4 1:0.27871845 2:0.968462 3:0.47951813 4:0.28255085 5:0.35504455 \n-4 1:0.48632108 2:0.968462 3:0.31026322 4:0.54522516 5:0.63113419 \n-4 1:0.64489737 2:0.968462 3:0.18002294 4:0.73105721 5:0.79964277 \n-4 1:0.77484729 2:0.968462 3:0.089514213 4:0.84205855 5:0.88605804 \n-1.727341234172313 1:1.110223e-16 2:0.968462 3:0.62188315 4:0.0053330527 \n-4 1:0.29417977 2:0.968462 3:0.43561511 4:0.32100884 5:0.38897225 \n-4 1:0.49686591 2:0.968462 3:0.28173902 4:0.58790345 5:0.66923055 \n0.9751911266566995 1:0.65667202 2:0.968462 3:0.16588218 4:0.77450348 5:0.83570467 \n-3.095997396956162 1:0.7896287 2:0.968462 3:0.080074824 4:0.86970938 5:0.90886342 \n4 1:0.35006785 2:1 3:0.82970816 4:0.38030279 5:0.47851904 \n4 1:0.53514209 2:1 3:0.58868438 4:0.6017341 5:0.68737238 \n4 1:0.71507251 2:1 3:0.33560967 4:0.80164848 5:0.85703099 \n4 1:0.79447844 2:1 3:0.21473718 4:0.88445319 5:0.92121171 \n4 1:0.89762061 2:1 3:0.11540479 4:0.9282506 5:0.94522236 \n4 1:0.35074595 2:1 3:0.80033147 4:0.38547741 5:0.47652111 \n4 1:0.55248203 2:1 3:0.56996523 4:0.62897564 5:0.70815781 \n4 1:0.73968257 2:1 3:0.31427182 4:0.83749631 5:0.88325545 \n4 1:0.81507275 2:1 3:0.19774749 4:0.91418716 5:0.94223302 \n4 1:0.91770545 2:1 3:0.10164741 4:0.94471065 5:0.95618294 \n-4 1:0.63636169 2:0.60396085 3:0.99458103 4:0.67745459 5:0.72835488 \n-4 1:0.73808876 2:0.60396085 3:0.75721796 4:0.78427235 5:0.82165028 \n-4 1:0.8272471 2:0.60396085 3:0.53372725 4:0.85930057 5:0.88728908 \n-4 1:0.88168328 2:0.60396085 3:0.38036571 4:0.90196676 5:0.92173156 \n-4 1:0.93904706 2:0.60396085 3:0.2155621 4:0.94138755 5:0.95273667 \n-4 1:0.63308991 2:0.60396085 3:1 4:0.74645801 5:0.78867746 \n-3.233273237004985 1:0.72232225 2:0.60396085 3:0.75818149 4:0.84859773 5:0.88102612 \n-4 1:0.84926449 2:0.60396085 3:0.51145537 4:0.91253379 5:0.92870259 \n-4 1:0.90193957 2:0.60396085 3:0.36511013 4:0.95489909 5:0.9634361 \n4 1:0.95800291 2:0.60396085 3:0.19894372 4:0.98477365 5:0.98641077 \n4 1:0.68965514 2:0.31741336 3:0.2038152 4:0.73049562 5:0.77815923 \n4 1:0.77513638 2:0.31741336 3:0.14055724 4:0.8006223 5:0.83882717 \n4 1:0.83919208 2:0.31741336 3:0.086713948 4:0.86038234 5:0.88890317 \n-4 1:0.90352909 2:0.31741336 3:0.034552692 4:0.91423216 5:0.932522 \n4 1:0.71515081 2:0.31741336 3:0.22376172 4:0.76051573 5:0.79817371 \n4 1:0.80623331 2:0.31741336 3:0.14059195 4:0.86160538 5:0.8872236 \n4 1:0.86861805 2:0.31741336 3:0.088410158 4:0.88882374 5:0.90852011 \n4 1:0.93115579 2:0.31741336 3:0.035735599 4:0.94354045 5:0.95350501 \n-4 1:0.97598135 2:0.31741336 3:0.0021718747 4:0.97336407 5:0.97710292 \n4 1:0.71255549 2:0.15366261 3:0.19665238 4:0.80866018 5:0.84884245 \n4 1:0.78720817 2:0.15366261 3:0.1400817 4:0.86454933 5:0.89488869 \n4 1:0.85478373 2:0.15366261 3:0.088991096 4:0.9025166 5:0.92499867 \n-4 1:0.91126304 2:0.15366261 3:0.044693607 4:0.93937849 5:0.95304189 \n-4 1:0.94869324 2:0.15366261 3:0.012039687 4:0.96113749 5:0.96887862 \n4 1:0.69859952 2:0.15366261 3:0.2200826 4:0.85610049 5:0.89220649 \n4 1:0.80326825 2:0.15366261 3:0.15498982 4:0.8952381 5:0.9177477 \n4 1:0.88632123 2:0.15366261 3:0.095820665 4:0.95420562 5:0.96516936 \n-4 1:0.93719134 2:0.15366261 3:0.054447094 4:0.9848587 5:0.98866872 \n4 1:0.4957595 2:0.61682917 3:0.23627649 4:0.58929455 5:0.65515728 \n-4 1:0.62845001 2:0.61682917 3:0.15815832 4:0.7230333 5:0.7772504 \n-4 1:0.76216783 2:0.61682917 3:0.082946168 4:0.83292226 5:0.86769002 \n-4 1:0.83940298 2:0.61682917 3:0.04375467 4:0.89518079 5:0.91712245 \n-4 1:0.8866763 2:0.61682917 3:0.016841206 4:0.92635005 5:0.94115568 \n4 1:0.56383136 2:0.61682917 3:0.23938811 4:0.67379575 5:0.72695226 \n1.024808873343301 1:0.69533731 2:0.61682917 3:0.15729644 4:0.80728098 5:0.84549458 \n-4 1:0.81962278 2:0.61682917 3:0.08073912 4:0.89085719 5:0.91261515 \n4 1:0.88213669 2:0.61682917 3:0.041955408 4:0.93539236 5:0.94654248 \n", "feature_dict": { "FUNQUE_feature": [ - "wd_essim", - "dlm", - "motion", + "ssim_scale0", + "ssim_scale1", + "ssim_scale2", + "ssim_scale3", + "adm_score", + "adm_scale0", + "adm_scale1", + "adm_scale2", + "adm_scale3", + "vif_score", + "vif_scale0", "vif_scale1", - "vif_scale2" + "vif_scale2", + "vif_scale3", + "strred_scale0", + "strred_scale1", + "strred_scale2", + "strred_scale3" ] }, "score_clip": [