Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simply metadata propagation API. #1407

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Simply metadata propagation API. #1407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3e54f66
fix: bugfix for adding/removing models.
Feb 2, 2025
556a7cb
refactor: cleanup unnecessary fields
Feb 1, 2025
dd4d749
refactor: Move tests from `test_predict.c` to `test_propagate_metadat…
Feb 1, 2025
1246b72
libvmaf: Implement non monotonic feature propagation.
Feb 1, 2025
a696d7c
refactor: remove feature name from VmafMetadataConfiguration
Feb 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion libvmaf/include/libvmaf/libvmaf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,6 @@ typedef struct VmafMetadata {
* @param data User data to pass to the callback.
*/
typedef struct VmafMetadataConfiguration {
char *feature_name;
void (*callback)(void *data, VmafMetadata *metadata);
void *data;
} VmafMetadataConfiguration;
Expand Down
117 changes: 72 additions & 45 deletions libvmaf/src/feature/feature_collector.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
#include "log.h"
#include "predict.h"

#define MAX(a, b) ((a) > (b) ? (a) : (b))

static int aggregate_vector_init(AggregateVector *aggregate_vector)
{
if (!aggregate_vector) return -EINVAL;
Expand Down Expand Up @@ -242,17 +244,19 @@ int vmaf_feature_collector_mount_model(VmafFeatureCollector *feature_collector,

VmafPredictModel *m = malloc(sizeof(VmafPredictModel));
if (!m) return -ENOMEM;

m->model = model;
m->next = NULL;

VmafPredictModel **head = &feature_collector->models;
while (*head && (*head)->next != NULL)
*head = (*head)->next;

if (!(*head))
*head = m;
else
(*head)->next = m;
m->last_highest_seen_index = 0;
m->last_lowest_seen_index = 0;

VmafPredictModel *head = feature_collector->models;
if (!head) {
feature_collector->models = m;
} else {
while (head->next) head = head->next;
head->next = m;
}

return 0;
}
Expand All @@ -263,24 +267,30 @@ int vmaf_feature_collector_unmount_model(VmafFeatureCollector *feature_collector
if (!feature_collector) return -EINVAL;
if (!model) return -EINVAL;

VmafPredictModel **head = &feature_collector->models;
while (*head && (*head)->model != model)
head = &(*head)->next;
VmafPredictModel *head = feature_collector->models;
VmafPredictModel *prev = NULL;

if (!(*head)) return -EINVAL;

VmafPredictModel *m = *head;
*head = m->next;
free(m);
while (head) {
if (head->model == model) {
if (prev) {
prev->next = head->next;
} else {
feature_collector->models = head->next;
}
free(head);
return 0;
}
prev = head;
head = head->next;
}

return 0;
return -ENOENT;
}

int vmaf_feature_collector_register_metadata(VmafFeatureCollector *feature_collector,
VmafMetadataConfiguration metadata_cfg)
{
if (!feature_collector) return -EINVAL;
if (!metadata_cfg.feature_name) return -EINVAL;
if (!metadata_cfg.callback) return -EINVAL;

VmafCallbackList *metadata = feature_collector->metadata;
Expand Down Expand Up @@ -347,30 +357,12 @@ int vmaf_feature_collector_append(VmafFeatureCollector *feature_collector,

int res = 0;

VmafCallbackItem *metadata_iter = feature_collector->metadata ?
feature_collector->metadata->head : NULL;
while (metadata_iter) {
// Check current feature name is the same as the metadata feature name
if (!strcmp(metadata_iter->metadata_cfg.feature_name, feature_name)) {

// Call the callback function with the metadata feature name
VmafMetadata data = {
.feature_name = metadata_iter->metadata_cfg.feature_name,
.picture_index = picture_index,
.score = score,
};
metadata_iter->metadata_cfg.callback(metadata_iter->metadata_cfg.data, &data);
// Move to the next metadata
goto next_metadata;
}

VmafPredictModel *model_iter = feature_collector->models;
VmafPredictModel *model_iter = feature_collector->models;

// If metadata feature name is not the same as the current feature feature_name
// Check if metadata feature name is the predicted feature
while (model_iter) {
VmafModel *model = model_iter->model;
while (model_iter && feature_collector->metadata->cnt) {
VmafModel *model = model_iter->model;

if (strcmp(model->name, feature_name)) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do these callbacks only work for model predicted metrics now? What about something like psnr?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes they are working only for predicted metrics currently. We can make it work for all metrics with such a change like below. However for make it work with those metrics also needs a registered vmaf model. So user can't only get psnr. Is this a thing that we need to rethink about?

https://github.com/yigithanyigit/vmaf/blob/a696d7cb22a54568c97c9f0562b6c34142a89acf/libvmaf/src/feature/feature_collector.c#L395-L406

to

                        if (!feature_collector->feature_vector[i]->score[process_index].written) {
                            frame_ready = false;
                            break;
                        }
                        data.feature_name = feature_collector->feature_vector[i]->name;
                        data.picture_index = process_index;
                        data.score = feature_collector->feature_vector[i]->score[process_index].value;

                        metadata_iter->metadata_cfg.callback(metadata_iter->metadata_cfg.data, &data);
                    }

pthread_mutex_unlock(&(feature_collector->lock));
res = vmaf_feature_collector_get_score(feature_collector,
model->name, &score, picture_index);
Expand All @@ -382,11 +374,46 @@ int vmaf_feature_collector_append(VmafFeatureCollector *feature_collector,
picture_index, &score, true, true, 0);
pthread_mutex_lock(&(feature_collector->lock));
}
model_iter = model_iter->next;
}
} else {
model_iter->last_highest_seen_index = MAX(model_iter->last_highest_seen_index, picture_index);
unsigned process_index = model_iter->last_lowest_seen_index;
double temp_score = 0.0;
bool frame_ready = true;

next_metadata:
metadata_iter = metadata_iter->next;
while (process_index <= model_iter->last_highest_seen_index && frame_ready) {
pthread_mutex_unlock(&(feature_collector->lock));
res = vmaf_feature_collector_get_score(feature_collector,
model->name, &temp_score, process_index);
frame_ready = !res;
pthread_mutex_lock(&(feature_collector->lock));

VmafCallbackItem *metadata_iter = feature_collector->metadata->cnt ?
feature_collector->metadata->head : NULL;

while (metadata_iter && frame_ready) {
VmafMetadata data = {0};
for (unsigned i = 0; i < model->n_features; i++) {
data.feature_name = model->feature[i].name;
data.picture_index = process_index;
data.score = find_feature_vector(feature_collector, model->feature[i].name)->score[process_index].value;

metadata_iter->metadata_cfg.callback(metadata_iter->metadata_cfg.data, &data);
}
data.feature_name = model->name;
data.picture_index = process_index;
data.score = temp_score;
metadata_iter->metadata_cfg.callback(metadata_iter->metadata_cfg.data, &data);

metadata_iter = metadata_iter->next;
}

if (frame_ready) {
process_index++;
model_iter->last_lowest_seen_index = process_index;
}
}
}
model_iter = model_iter->next;
}

unlock:
Expand Down
2 changes: 2 additions & 0 deletions libvmaf/src/feature/feature_collector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ typedef struct {

typedef struct VmafPredictModel {
VmafModel *model;
unsigned last_highest_seen_index;
unsigned last_lowest_seen_index;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you explain why these index tracking fields are needed?

Copy link
Contributor Author

@yigithanyigit yigithanyigit Feb 3, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Of course, as you know if application is multithreaded model scores might be calculated in out of order. Those indexes basically helping them to make in order. I am using those indexes for the current calculated index is in order. For example;

Let's say we have frame order like this:

3, 0, 1, 2

Frame 3 arrives -> Calculate but not propagate, Update last_highest_seen_index to 3;
Propagated frame -> None

Frame 0 arrives -> Calculate and propagate since frame index (0) == last_lowest_seen_index. Increase last_lowest_seen_index by 1. It tries to propagate until last_highest_seen_index (3) but since there is no Frame 1 it stops.
Propagated frame -> Frame 0

Frame 1 arrives -> Calculate and propagate since frame index (1) == last_lowest_seen_index. Increase last_lowest_seen_index by 1. It tries to propagate until last_highest_seen_index but since there is no Frame 2 it stops.
Propagated frame -> Frame 1

Frame 2 arrives -> Calculate and propagate since frame index (2) == last_lowest_seen_index. Increase last_lowest_seen_index by 1. It tries to propagate until last_highest_seen_index and we have ready Frame 3 so also it propagates that frame.
Propagated frame -> Frame 2, Frame 3

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it a problem if the callbacks come out of order? Using your method, callbacks could be withheld for an unlimited number of frames (1 - N) until frame 0 is ready?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it a problem if the callbacks come out of order?

I don't think it's a problem for us. But for the sake of usability of API I think we need something like this. Otherwise users might need to reorder the frames. For example I am thinking FFmpeg, in that implementation we need to send frames in order to filtergraph (If we want to write features to original frame's metadata otherwise we don't need).

Using your method, callbacks could be withheld for an unlimited number of frames (1 - N) until frame 0 is ready?

Unfortunately, yes. Is there a possibility that frame 0 somehow not calculated? I thought this situation while implementing this and I concluded if somehow one of the frames doesn't calculate then we can't calculate any other future frames due to metrics like Motion right? So I ended up VMAF can't drop frames.

struct VmafPredictModel *next;
} VmafPredictModel;

Expand Down
3 changes: 3 additions & 0 deletions libvmaf/src/metadata_handler.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ int vmaf_metadata_init(VmafCallbackList **const metadata)
if (!metadata_s) goto fail;

metadata_s->head = NULL;
metadata_s->cnt = 0;

return 0;

Expand All @@ -56,6 +57,8 @@ int vmaf_metadata_append(VmafCallbackList *metadata, const VmafMetadataConfigura
iter->next = node;
}

metadata->cnt++;

return 0;

fail:
Expand Down
3 changes: 1 addition & 2 deletions libvmaf/src/metadata_handler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,12 @@

typedef struct VmafCallbackItem {
VmafMetadataConfiguration metadata_cfg;
void (*callback)(void *, VmafMetadata *);
void *data;
struct VmafCallbackItem *next;
} VmafCallbackItem;

typedef struct VmafCallbackList{
VmafCallbackItem *head;
unsigned cnt;
} VmafCallbackList;

int vmaf_metadata_init(VmafCallbackList **const metadata);
Expand Down
9 changes: 6 additions & 3 deletions libvmaf/test/meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@ test_picture = executable('test_picture',
)

test_propagate_metadata = executable('test_propagate_metadata',
['test.c', 'test_propagate_metadata.c', '../src/metadata_handler.c'],
['test.c', 'test_propagate_metadata.c', '../src/metadata_handler.c','../src/dict.c',
'../src/feature/feature_collector.c', '../src/feature/alias.c', '../src/model.c', '../src/svm.cpp', '../src/log.c',
'../src/read_json_model.c', '../src/pdjson.c', json_model_c_sources, '../src/feature/feature_name.c', '../src/feature/feature_extractor.c'],
include_directories : [libvmaf_inc, test_inc, include_directories('../src/')],
link_with : get_option('default_library') == 'both' ? libvmaf.get_static_lib() : libvmaf,
)

test_feature_collector = executable('test_feature_collector',
Expand Down Expand Up @@ -50,8 +53,8 @@ test_model = executable('test_model',
)

test_predict = executable('test_predict',
['test.c', 'test_predict.c', '../src/dict.c', '../src/metadata_handler.c',
'../src/feature/feature_collector.c', '../src/feature/alias.c', '../src/model.c', '../src/svm.cpp', '../src/log.c',
['test.c', 'test_predict.c', '../src/dict.c', '../src/feature/feature_collector.c',
'../src/feature/alias.c', '../src/model.c', '../src/svm.cpp', '../src/log.c',
'../src/read_json_model.c', '../src/pdjson.c', json_model_c_sources, '../src/feature/feature_name.c', '../src/feature/feature_extractor.c',],
include_directories : [libvmaf_inc, test_inc, include_directories('../src/')],
link_with : get_option('default_library') == 'both' ? libvmaf.get_static_lib() : libvmaf,
Expand Down
109 changes: 93 additions & 16 deletions libvmaf/test/test_feature_collector.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,21 +57,58 @@ static char* test_model_mount()
err = vmaf_feature_collector_init(&feature_collector);
mu_assert("problem during vmaf_feature_collector_init", !err);

VmafModelConfig model_cfg = { 0 };
VmafModel *model;
vmaf_model_load(&model, &model_cfg, "vmaf_v0.6.1");
mu_assert("problem during vmaf_model_load", model);
char *model_names[] = {"vmaf_0", "vmaf_1", "vmaf_2"};

VmafModelConfig model_cfg = {
.name = model_names[0],
.flags = VMAF_MODEL_FLAGS_DEFAULT,
};

VmafModelConfig model_cfg1 = {
.name = model_names[1],
.flags = VMAF_MODEL_FLAGS_DEFAULT,
};

err = vmaf_feature_collector_mount_model(feature_collector, model);
VmafModelConfig model_cfg2 = {
.name = model_names[2],
.flags = VMAF_MODEL_FLAGS_DEFAULT,
};

VmafModel *model0, *model1, *model2;
vmaf_model_load(&model0, &model_cfg, "vmaf_v0.6.1");
mu_assert("problem during vmaf_model_load", model0);

vmaf_model_load(&model1, &model_cfg1, "vmaf_v0.6.1");
mu_assert("problem during vmaf_model_load", model0);

vmaf_model_load(&model2, &model_cfg2, "vmaf_v0.6.1");
mu_assert("problem during vmaf_model_load", model0);

err = vmaf_feature_collector_mount_model(feature_collector, model0);
mu_assert("problem during vmaf_model_mount",
feature_collector->models);

err = vmaf_feature_collector_mount_model(feature_collector, model);
err = vmaf_feature_collector_mount_model(feature_collector, model1);
mu_assert("problem during vmaf_model_mount",
feature_collector->models->next);

vmaf_model_destroy(model);
err = vmaf_feature_collector_mount_model(feature_collector, model2);
mu_assert("problem during vmaf_model_mount",
feature_collector->models->next);

VmafPredictModel *model_iter = feature_collector->models;
unsigned i = 0;

while (model_iter) {
mu_assert("model name does not match", !strcmp(model_iter->model->name,
model_names[i]));
model_iter = model_iter->next;
i++;
}

vmaf_model_destroy(model0);
vmaf_model_destroy(model1);
vmaf_model_destroy(model2);
vmaf_feature_collector_destroy(feature_collector);

return NULL;
Expand All @@ -85,20 +122,60 @@ static char* test_model_unmount()
err = vmaf_feature_collector_init(&feature_collector);
mu_assert("problem during vmaf_feature_collector_init", !err);

VmafModelConfig model_cfg = { 0 };
VmafModel *model;
vmaf_model_load(&model, &model_cfg, "vmaf_v0.6.1");
mu_assert("problem during vmaf_model_load", model);
char *model_names[] = {"vmaf_0", "vmaf_1", "vmaf_2"};

VmafModelConfig model_cfg = {
.name = model_names[0],
.flags = VMAF_MODEL_FLAGS_DEFAULT,
};

VmafModelConfig model_cfg1 = {
.name = model_names[1],
.flags = VMAF_MODEL_FLAGS_DEFAULT,
};

VmafModelConfig model_cfg2 = {
.name = model_names[2],
.flags = VMAF_MODEL_FLAGS_DEFAULT,
};

VmafModel *model0, *model1, *model2;
vmaf_model_load(&model0, &model_cfg, "vmaf_v0.6.1");
mu_assert("problem during vmaf_model_load", model0);

err = vmaf_feature_collector_mount_model(feature_collector, model);
vmaf_model_load(&model1, &model_cfg1, "vmaf_v0.6.1");
mu_assert("problem during vmaf_model_load", model0);

vmaf_model_load(&model2, &model_cfg2, "vmaf_v0.6.1");
mu_assert("problem during vmaf_model_load", model0);

err = vmaf_feature_collector_mount_model(feature_collector, model0);
mu_assert("problem during vmaf_model_mount",
feature_collector->models);

err = vmaf_feature_collector_unmount_model(feature_collector, model);
mu_assert("problem during vmaf_model_unmount",
!feature_collector->models);
err = vmaf_feature_collector_mount_model(feature_collector, model1);
mu_assert("problem during vmaf_model_mount",
feature_collector->models->next);

vmaf_model_destroy(model);
err = vmaf_feature_collector_mount_model(feature_collector, model2);
mu_assert("problem during vmaf_model_mount",
feature_collector->models->next);

err = vmaf_feature_collector_unmount_model(feature_collector, model0);
mu_assert("problem during vmaf_model_unmount", !err);

err = vmaf_feature_collector_unmount_model(feature_collector, model1);
mu_assert("problem during vmaf_model_unmount", !err);

err = vmaf_feature_collector_unmount_model(feature_collector, model2);
mu_assert("problem during vmaf_model_unmount", !err);

mu_assert("feature_collector->models should be NULL",
!feature_collector->models);

vmaf_model_destroy(model0);
vmaf_model_destroy(model1);
vmaf_model_destroy(model2);
vmaf_feature_collector_destroy(feature_collector);

return NULL;
Expand Down
Loading
Loading