Skip to content

Commit dc5d04f

Browse files
author
Martin Bruse
committed
Added more datasets and metrics to the analysis.
- Added missing includes. - Added more garbage to .gitignore. - Added more doc comments. - Fixed many small bugs. - Fixed wording in various places. - Made the calculation tool optionally able to skip calculating already-existing metrics. - Added leaderboard flag to the analysis tool that shows top list of metrics. - Made the table output be Markdown-compatible. - Added an optional error handler to the worker pool. - Added many more external metrics. - Added ingestion tool for more datasets.
1 parent 645cfd1 commit dc5d04f

File tree

10 files changed

+924
-272
lines changed

10 files changed

+924
-272
lines changed

.gitignore

+5-1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,8 @@ debug_build
33
asan_build
44
.vscode
55
python/__pycache__/**
6-
**/METADATA
6+
**/METADATA
7+
# Some files produced by one of the external metrics.
8+
pesq_results.txt
9+
analized
10+
Testing/*

cpp/zimt/goohrli.cc

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include <cstddef>
1919
#include <utility>
2020

21+
#include "absl/status/statusor.h"
2122
#include "absl/types/span.h"
2223
#include "hwy/aligned_allocator.h"
2324
#include "hwy/base.h"

go/bin/perceptual_audio/perceptual_audio.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
// limitations under the License.
1414

1515
// perceptual_audio creates a study from https://github.com/pranaymanocha/PerceptualAudio/blob/master/dataset/README.md.
16+
//
17+
// Download and unpack the dataset ZIP, and provide the unpacked directory
18+
// as -source when running this binary.
1619
package main
1720

1821
import (
@@ -56,7 +59,7 @@ func populate(source string, dest string, workers int) error {
5659

5760
lineReader := bufio.NewReader(res.Body)
5861
err = nil
59-
bar := progress.New("Recoding")
62+
bar := progress.New("Transcoding")
6063
pool := worker.Pool[*data.Reference]{
6164
Workers: workers,
6265
OnChange: bar.Update,
@@ -95,10 +98,10 @@ func populate(source string, dest string, workers int) error {
9598
})
9699
lineIndex++
97100
}
98-
bar.Finish()
99101
if err := pool.Error(); err != nil {
100102
log.Println(err.Error())
101103
}
104+
bar.Finish()
102105
refs := []*data.Reference{}
103106
for ref := range pool.Results() {
104107
refs = append(refs, ref)
@@ -112,7 +115,7 @@ func populate(source string, dest string, workers int) error {
112115
func main() {
113116
source := flag.String("source", "", "Directory containing the unpacked http://percepaudio.cs.princeton.edu/icassp2020_perceptual/audio_perception.zip.")
114117
destination := flag.String("dest", "", "Destination directory.")
115-
workers := flag.Int("workers", runtime.NumCPU(), "Number of workers downloading sounds.")
118+
workers := flag.Int("workers", runtime.NumCPU(), "Number of workers transcoding sounds.")
116119
flag.Parse()
117120
if *source == "" || *destination == "" {
118121
flag.Usage()

go/bin/score/score.go

+30-2
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ import (
2121
"fmt"
2222
"log"
2323
"os"
24+
"path/filepath"
2425
"runtime"
26+
"sort"
2527

2628
"github.com/google/zimtohrli/go/data"
2729
"github.com/google/zimtohrli/go/goohrli"
@@ -37,22 +39,42 @@ const (
3739
func main() {
3840
details := flag.String("details", "", "Path to database directory with a study to show the details from.")
3941
calculate := flag.String("calculate", "", "Path to a database directory with a study to calculate metrics for.")
42+
force := flag.Bool("force", false, "Whether to recalculate scores that already exist.")
4043
calculateZimtohrli := flag.Bool("calculate_zimtohrli", false, "Whether to calculate Zimtohrli scores.")
4144
calculateViSQOL := flag.Bool("calculate_visqol", false, "Whether to calculate ViSQOL scores.")
4245
calculatePipeMetric := flag.String("calculate_pipe", "", "Path to a binary that serves metrics via stdin/stdout pipe. Install some of the via 'install_python_metrics.py'.")
4346
zimtohrliFrequencyResolution := flag.Float64("zimtohrli_frequency_resolution", goohrli.DefaultFrequencyResolution(), "Smallest bandwidth of the Zimtohrli filterbank.")
4447
zimtohrliPerceptualSampleRate := flag.Float64("zimtohrli_perceptual_sample_rate", goohrli.DefaultPerceptualSampleRate(), "Sample rate of the Zimtohrli spectrograms.")
4548
correlate := flag.String("correlate", "", "Path to a database directory with a study to correlate scores for.")
49+
leaderboard := flag.String("leaderboard", "", "Glob to directories with databases to compute leaderboard for.")
4650
accuracy := flag.String("accuracy", "", "Path to a database directory with a study to provide JND accuracy for.")
4751
workers := flag.Int("workers", runtime.NumCPU(), "Number of concurrent workers for tasks.")
4852
failFast := flag.Bool("fail_fast", false, "Whether to panic immediately on any error.")
4953
flag.Parse()
5054

51-
if *details == "" && *calculate == "" && *correlate == "" && *accuracy == "" {
55+
if *details == "" && *calculate == "" && *correlate == "" && *accuracy == "" && *leaderboard == "" {
5256
flag.Usage()
5357
os.Exit(1)
5458
}
5559

60+
if *leaderboard != "" {
61+
databases, err := filepath.Glob(*leaderboard)
62+
if err != nil {
63+
log.Fatal(err)
64+
}
65+
studies := make(data.Studies, len(databases))
66+
for index, path := range databases {
67+
if studies[index], err = data.OpenStudy(path); err != nil {
68+
log.Fatal(err)
69+
}
70+
}
71+
board, err := studies.Leaderboard()
72+
if err != nil {
73+
log.Fatal(err)
74+
}
75+
fmt.Println(board)
76+
}
77+
5678
if *details != "" {
5779
study, err := data.OpenStudy(*details)
5880
if err != nil {
@@ -107,7 +129,13 @@ func main() {
107129
log.Print("No metrics to calculate, provide one of the -calculate_XXX flags!")
108130
os.Exit(2)
109131
}
110-
if err := study.Calculate(measurements, pool); err != nil {
132+
sortedTypes := sort.StringSlice{}
133+
for scoreType := range measurements {
134+
sortedTypes = append(sortedTypes, string(scoreType))
135+
}
136+
sort.Sort(sortedTypes)
137+
log.Printf("*** Calculating %+v (force=%v)", sortedTypes, *force)
138+
if err := study.Calculate(measurements, pool, *force); err != nil {
111139
log.Fatal(err)
112140
}
113141
bar.Finish()

go/bin/sebass_db/sebass_db.go

+163
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
// Copyright 2024 The Zimtohrli Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// sebass_db creates a study from https://www.audiolabs-erlangen.de/resources/2019-WASPAA-SEBASS/.
16+
//
17+
// It currently supports SASSEC, SiSEC08, SAOC, and PEASS-DB. SiSEC18 at
18+
// that web site doesn't contain all audio, and is not currently supported.
19+
//
20+
// Download and unpac one of the supported ZIP files, and use the directory
21+
// it unpacked as -source when running this binary.
22+
package main
23+
24+
import (
25+
"encoding/csv"
26+
"flag"
27+
"fmt"
28+
"log"
29+
"math"
30+
"os"
31+
"path/filepath"
32+
"reflect"
33+
"runtime"
34+
"strconv"
35+
36+
"github.com/google/zimtohrli/go/aio"
37+
"github.com/google/zimtohrli/go/data"
38+
"github.com/google/zimtohrli/go/progress"
39+
"github.com/google/zimtohrli/go/worker"
40+
)
41+
42+
func populate(source string, dest string, workers int, failFast bool) error {
43+
study, err := data.OpenStudy(dest)
44+
if err != nil {
45+
return err
46+
}
47+
defer study.Close()
48+
49+
csvFiles, err := filepath.Glob(filepath.Join(source, "*.csv"))
50+
if err != nil {
51+
return err
52+
}
53+
for _, csvFile := range csvFiles {
54+
signals := "Signals"
55+
switch filepath.Base(csvFile) {
56+
case "SAOC_1_anonymized.csv":
57+
signals = "Signals_1"
58+
case "SAOC_2_anonymized.csv":
59+
signals = "Signals_2"
60+
case "SAOC_3_anonymized.csv":
61+
signals = "Signals_3"
62+
}
63+
fileReader, err := os.Open(csvFile)
64+
if err != nil {
65+
return err
66+
}
67+
defer fileReader.Close()
68+
csvReader := csv.NewReader(fileReader)
69+
header, err := csvReader.Read()
70+
if err != nil {
71+
return err
72+
}
73+
if !reflect.DeepEqual(header, []string{"Testname", "Listener", "Trial", "Condition", "Ratingscore"}) {
74+
return fmt.Errorf("header %+v doesn't match expected SEBASS-DB header", header)
75+
}
76+
err = nil
77+
bar := progress.New(fmt.Sprintf("Transcoding from %q", csvFile))
78+
pool := worker.Pool[*data.Reference]{
79+
Workers: workers,
80+
OnChange: bar.Update,
81+
FailFast: failFast,
82+
}
83+
var loopLine []string
84+
lineIndex := 0
85+
for loopLine, err = csvReader.Read(); err == nil; loopLine, err = csvReader.Read() {
86+
line := loopLine
87+
if len(line) == 0 {
88+
continue
89+
}
90+
if line[3] == "anchor" {
91+
line[3] = "anker_mix"
92+
}
93+
if line[3] == "hidden_ref" {
94+
line[3] = "orig"
95+
}
96+
if line[3] == "SAOC" {
97+
continue
98+
}
99+
mos, err := strconv.ParseFloat(line[4], 64)
100+
if err != nil {
101+
return err
102+
}
103+
if math.IsNaN(mos) {
104+
continue
105+
}
106+
refIndex := lineIndex
107+
pool.Submit(func(f func(*data.Reference)) error {
108+
ref := &data.Reference{
109+
Name: fmt.Sprintf("ref-%v", refIndex),
110+
}
111+
var err error
112+
path := filepath.Join(source, signals, "orig", fmt.Sprintf("%s.wav", line[2]))
113+
ref.Path, err = aio.Recode(path, dest)
114+
if err != nil {
115+
return fmt.Errorf("unable to fetch %q", path)
116+
}
117+
dist := &data.Distortion{
118+
Name: fmt.Sprintf("dist-%v", refIndex),
119+
Scores: map[data.ScoreType]float64{
120+
data.MOS: mos,
121+
},
122+
}
123+
path = filepath.Join(source, signals, line[3], fmt.Sprintf("%s.wav", line[2]))
124+
dist.Path, err = aio.Recode(path, dest)
125+
if err != nil {
126+
return fmt.Errorf("unable to fetch %q", path)
127+
}
128+
ref.Distortions = append(ref.Distortions, dist)
129+
f(ref)
130+
return nil
131+
})
132+
lineIndex++
133+
}
134+
if err := pool.Error(); err != nil {
135+
log.Println(err.Error())
136+
}
137+
bar.Finish()
138+
refs := []*data.Reference{}
139+
for ref := range pool.Results() {
140+
refs = append(refs, ref)
141+
}
142+
if err := study.Put(refs); err != nil {
143+
return err
144+
}
145+
}
146+
return nil
147+
}
148+
149+
func main() {
150+
source := flag.String("source", "", "Directory containing one of the unpacked datasets from https://www.audiolabs-erlangen.de/resources/2019-WASPAA-SEBASS/.")
151+
destination := flag.String("dest", "", "Destination directory.")
152+
workers := flag.Int("workers", runtime.NumCPU(), "Number of workers transcoding sounds.")
153+
failFast := flag.Bool("fail_fast", false, "Whether to exit immediately at the first error.")
154+
flag.Parse()
155+
if *source == "" || *destination == "" {
156+
flag.Usage()
157+
os.Exit(1)
158+
}
159+
160+
if err := populate(*source, *destination, *workers, *failFast); err != nil {
161+
log.Fatal(err)
162+
}
163+
}

0 commit comments

Comments
 (0)