Skip to content

Commit cf4ee3f

Browse files
author
Jon Fiscus
committedAug 4, 2022
This commit is a bug test case for the asclite scorer when the
intersegment gaps are expected to be added to the scoring output but they are not. When the isgbug-work.ctm file is scored, inter-segment-gaps are produced but for the isgbug-fail.ctm file, the inter-segment-gaps are not produced. This feels like a condition problem somewhere but I'm not sure were. To run the test, cd into src/asclite/testfiles and run: perl ./ascliteTest.pl -s todebug While the test runs successfully, it should not. The ascliteTestOutDir/isgbug-fail.sgml should have inter-segment-gaps like ascliteTestOutDir/isgbug-work.sgml.
1 parent d4a0516 commit cf4ee3f

File tree

6 files changed

+159
-2
lines changed

6 files changed

+159
-2
lines changed
 

‎src/asclite/testfiles/ascliteTest.pl

+9-2
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
sub ok_quit { print(join(' ', @_), "\n"); &ok_exit(); }
3030
####################
3131

32-
my $Usage = "Usage: ascliteTest.pl -s (all|sastt|std|mdm04|mdm04ByFile|cts04|mdmVariations|passed|notpassed) [ -m ]\n";
32+
my $Usage = "Usage: ascliteTest.pl -s (all|sastt|std|mdm04|mdm04ByFile|cts04|mdmVariations|passed|notpassed|todebug) [ -m ]\n";
3333
my $suite = "std";
3434
my $bigMem = 0;
3535
my $result = GetOptions("s=s" => \$suite, "m" => \$bigMem);
@@ -90,6 +90,12 @@
9090
RunAscliteTest("generic-1", "-F -D -generic-cost -noisg", "", "-h generic-1.rttm rttm generic-1");
9191
}
9292

93+
if ($suite =~ /^(todebug)$/)
94+
{
95+
RunAscliteTest("isgbug-work", "-D -F -adaptive-cost -time-prune 400 -word-time-align 400 -memory-limit 8 -difficulty-limit 8 -overlap-limit 1 -memory-compression 256 -force-memory-compression", "-r isgbug.stm stm", "-h isgbug-work.ctm ctm");
96+
RunAscliteTest("isgbug-fail", "-D -F -adaptive-cost -time-prune 400 -word-time-align 400 -memory-limit 8 -difficulty-limit 8 -overlap-limit 1 -memory-compression 256 -force-memory-compression", "-r isgbug.stm stm", "-h isgbug-fail.ctm ctm");
97+
}
98+
9399
&error_quit("Errors Occured. Exiting with non-zero code") if ($failure);
94100
&ok_quit();
95101

@@ -100,7 +106,7 @@ sub RunCompatTest
100106
if (! -f "$compatOutDir/$testId.sgml")
101107
{
102108
print "Building Authoritative SGML file: $opts, $refOpts, $hypOpts\n";
103-
system "$scliteCom $opts $refOpts $hypOpts -o sgml stdout -f 0 | perl -pe 's/(creation_date=\")[^\"]+/\$1/' > $compatOutDir/$testId.sgml";
109+
system "$scliteCom $opts $refOpts $hypOpts -o sgml stdout -f 3 | perl -pe 's/(creation_date=\")[^\"]+/\$1/' > $compatOutDir/$testId.sgml";
104110
}
105111

106112
print "Comparing asclite to Authoritative SGML file: $opts, $refOpts, $hypOpts\n";
@@ -138,6 +144,7 @@ sub RunAscliteTest
138144
if (! -f "$ascliteTestOutDir/$testId.sgml")
139145
{
140146
print "Building Authoritative SGML file: $opts, $refOpts, $hypOpts\n";
147+
print "$ascliteCom $opts $refOpts $hypOpts -o sgml stdout -f 0 | perl -pe 's/(creation_date=\")[^\"]+/\$1/' > $ascliteTestOutDir/$testId.sgml";
141148
system "$ascliteCom $opts $refOpts $hypOpts -o sgml stdout -f 0 | perl -pe 's/(creation_date=\")[^\"]+/\$1/' > $ascliteTestOutDir/$testId.sgml";
142149
}
143150
else

‎src/asclite/testfiles/ascliteTestOutDir/isgbug-fail.sgml

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
<SYSTEM title="isgbug-fail.ctm" ref_fname="isgbug.stm" hyp_fname="" creation_date="" format="2.4" frag_corr="TRUE" opt_del="TRUE" weight_ali="FALSE" weight_filename="">
2+
<CATEGORY id="0" title="" desc="">
3+
</CATEGORY>
4+
<LABEL id="O" title="Overall" desc="">
5+
</LABEL>
6+
<CATEGORY id="1" title="Corpus" desc="">
7+
</CATEGORY>
8+
<LABEL id="12" title="Speech012" desc="">
9+
</LABEL>
10+
<CATEGORY id="2" title="Gender" desc="">
11+
</CATEGORY>
12+
<LABEL id="F" title="Female" desc="">
13+
</LABEL>
14+
<LABEL id="M" title="Male" desc="">
15+
</LABEL>
16+
<SPEAKER id="sp002">
17+
<PATH id="(sp002-000)" word_cnt="4" labels="<O,12,U>" file="Speech012" channel="0" sequence="0" R_T1="600.490" R_T2="602.150">
18+
D,"got",:D,"your",:D,"meeting",:D,"request",
19+
</PATH>
20+
<PATH id="(sp002-001)" word_cnt="29" labels="<O,12,U>" file="Speech012" channel="0" sequence="1" R_T1="649.020" R_T2="658.270">
21+
D,"yeah",:C,"(%partial)","":D,"i",:D,"would",:D,"love",:C,"(%partial)","":D,"to",:D,"grab",:D,"some",:D,"time",:D,"with",:D,"you",:C,"(%hesitation)","":D,"separately",:D,"and",:C,"(%hesitation)","":D,"get",:D,"ramped",:D,"up",:D,"and",:D,"also",:D,"maybe",:C,"(%partial)","":D,"get",:D,"my",:D,"hands",:D,"on",:D,"some",:D,"data",
22+
</PATH>
23+
<PATH id="(sp002-002)" word_cnt="17" labels="<O,12,U>" file="Speech012" channel="0" sequence="2" R_T1="671.000" R_T2="675.900">
24+
D,"would",:D,"you",:D,"mind",:D,"including",:D,"pavan",:C,"(%partial)","":D,"he",:D,"might",:D,"he",:D,"had",:D,"would",:D,"have",:D,"some",:C,"(%partial)","":D,"good",:D,"ideas",:D,"too",
25+
</PATH>
26+
<PATH id="(sp002-003)" word_cnt="15" labels="<O,12,U>" file="Speech012" channel="0" sequence="3" R_T1="677.200" R_T2="684.510">
27+
D,"our",:D,"new",:D,"data",:D,"scientist",:D,"i",:D,"want",:D,"him",:D,"to",:C,"(%partial)","":D,"to",:D,"work",:D,"that",:D,"those",:D,"muscles",:C,"(%laughter)",""
28+
</PATH>
29+
<PATH id="(sp002-004)" word_cnt="3" labels="<O,12,U>" file="Speech012" channel="0" sequence="4" R_T1="685.300" R_T2="686.920">
30+
D,"muscles",:D,"thank",:D,"you",
31+
</PATH>
32+
<PATH id="(sp002-005)" word_cnt="2" labels="<O,12,U>" file="Speech012" channel="0" sequence="5" R_T1="1617.620" R_T2="1619.950">
33+
C,"(%laughter)","":D,"no",
34+
</PATH>
35+
<PATH id="(sp002-006)" word_cnt="2" labels="<O,12,U>" file="Speech012" channel="0" sequence="6" R_T1="1874.480" R_T2="1875.730">
36+
D,"li",:D,"juan",
37+
</PATH>
38+
<PATH id="(sp002-007)" word_cnt="7" labels="<O,12,U>" file="Speech012" channel="0" sequence="7" R_T1="1876.300" R_T2="1880.310">
39+
D,"i",:D,"see",:D,"something",:D,"that",:D,"says",:D,"kpi",:D,"metrics",
40+
</PATH>
41+
<PATH id="(sp002-008)" word_cnt="7" labels="<O,12,U>" file="Speech012" channel="0" sequence="8" R_T1="1880.920" R_T2="1887.420">
42+
D,"is",:D,"there",:C,"(%hesitation)","":D,"other",:D,"kpi",:D,"listed",:D,"somewhere",
43+
</PATH>
44+
<PATH id="(sp002-009)" word_cnt="32" labels="<O,12,U>" file="Speech012" channel="0" sequence="9" R_T1="1922.340" R_T2="1931.720">
45+
D,"right",:D,"and",:D,"it",:D,"is",:D,"it",:D,"is",:D,"important",:D,"for",:D,"us",:C,"(%partial)","":D,"to",:D,"understand",:D,"what",:D,"their",:D,"kpi",:D,"is",:D,"because",:D,"that",:D,"could",:D,"have",:D,"an",:D,"impact",:D,"on",:D,"how",:D,"we",:D,"what",:D,"we",:D,"want",:D,"to",:D,"measure",:D,"or",:D,"how",
46+
</PATH>
47+
</SPEAKER>
48+
</SYSTEM>

‎src/asclite/testfiles/ascliteTestOutDir/isgbug-work.sgml

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
<SYSTEM title="isgbug-work.ctm" ref_fname="isgbug.stm" hyp_fname="isgbug-work.ctm" creation_date="" format="2.4" frag_corr="TRUE" opt_del="TRUE" weight_ali="FALSE" weight_filename="">
2+
<CATEGORY id="0" title="" desc="">
3+
</CATEGORY>
4+
<LABEL id="O" title="Overall" desc="">
5+
</LABEL>
6+
<CATEGORY id="1" title="Corpus" desc="">
7+
</CATEGORY>
8+
<LABEL id="12" title="Speech012" desc="">
9+
</LABEL>
10+
<CATEGORY id="2" title="Gender" desc="">
11+
</CATEGORY>
12+
<LABEL id="F" title="Female" desc="">
13+
</LABEL>
14+
<LABEL id="M" title="Male" desc="">
15+
</LABEL>
16+
<SPEAKER id="sp002">
17+
<PATH id="(sp002-000)" word_cnt="4" labels="<O,12,U>" file="Speech012" channel="0" sequence="0" R_T1="600.490" R_T2="602.150" word_aux="h_t1+t2,h_conf">
18+
D,"got",,,:D,"your",,,:D,"meeting",,,:D,"request",,,
19+
</PATH>
20+
<PATH id="(sp002-001)" word_cnt="29" labels="<O,12,U>" file="Speech012" channel="0" sequence="1" R_T1="649.020" R_T2="658.270" word_aux="h_t1+t2,h_conf">
21+
D,"yeah",,,:C,"(%partial)","",,:D,"i",,,:D,"would",,,:D,"love",,,:C,"(%partial)","",,:D,"to",,,:D,"grab",,,:D,"some",,,:D,"time",,,:D,"with",,,:D,"you",,,:C,"(%hesitation)","",,:D,"separately",,,:D,"and",,,:C,"(%hesitation)","",,:D,"get",,,:D,"ramped",,,:D,"up",,,:D,"and",,,:D,"also",,,:D,"maybe",,,:C,"(%partial)","",,:D,"get",,,:D,"my",,,:D,"hands",,,:D,"on",,,:D,"some",,,:D,"data",,,
22+
</PATH>
23+
<PATH id="(sp002-002)" word_cnt="17" labels="<O,12,U>" file="Speech012" channel="0" sequence="2" R_T1="671.000" R_T2="675.900" word_aux="h_t1+t2,h_conf">
24+
D,"would",,,:D,"you",,,:D,"mind",,,:D,"including",,,:D,"pavan",,,:C,"(%partial)","",,:D,"he",,,:D,"might",,,:D,"he",,,:D,"had",,,:D,"would",,,:D,"have",,,:D,"some",,,:C,"(%partial)","",,:D,"good",,,:D,"ideas",,,:D,"too",,,
25+
</PATH>
26+
<PATH id="(sp002-003)" word_cnt="15" labels="<O,12,U>" file="Speech012" channel="0" sequence="3" R_T1="677.200" R_T2="684.510" word_aux="h_t1+t2,h_conf">
27+
D,"our",,,:D,"new",,,:D,"data",,,:D,"scientist",,,:D,"i",,,:D,"want",,,:D,"him",,,:D,"to",,,:C,"(%partial)","",,:D,"to",,,:D,"work",,,:D,"that",,,:D,"those",,,:D,"muscles",,,:C,"(%laughter)","",,
28+
</PATH>
29+
<PATH id="(sp002-004)" word_cnt="3" labels="<O,12,U>" file="Speech012" channel="0" sequence="4" R_T1="685.300" R_T2="686.920" word_aux="h_t1+t2,h_conf">
30+
D,"muscles",,,:D,"thank",,,:D,"you",,,
31+
</PATH>
32+
<PATH id="(sp002-005)" word_cnt="2" labels="<O,12,U>" file="Speech012" channel="0" sequence="5" R_T1="1617.620" R_T2="1619.950" word_aux="h_t1+t2,h_conf">
33+
C,"(%laughter)","",,:D,"no",,,
34+
</PATH>
35+
<PATH id="(sp002-006)" word_cnt="2" labels="<O,12,U>" file="Speech012" channel="0" sequence="6" R_T1="1874.480" R_T2="1875.730" word_aux="h_t1+t2,h_conf">
36+
D,"li",,,:D,"juan",,,
37+
</PATH>
38+
<PATH id="(sp002-007)" word_cnt="7" labels="<O,12,U>" file="Speech012" channel="0" sequence="7" R_T1="1876.300" R_T2="1880.310" word_aux="h_t1+t2,h_conf">
39+
D,"i",,,:D,"see",,,:D,"something",,,:D,"that",,,:D,"says",,,:D,"kpi",,,:D,"metrics",,,
40+
</PATH>
41+
<PATH id="(sp002-008)" word_cnt="7" labels="<O,12,U>" file="Speech012" channel="0" sequence="8" R_T1="1880.920" R_T2="1887.420" word_aux="h_t1+t2,h_conf">
42+
D,"is",,,:D,"there",,,:C,"(%hesitation)","",,:D,"other",,,:D,"kpi",,,:D,"listed",,,:D,"somewhere",,,
43+
</PATH>
44+
<PATH id="(sp002-009)" word_cnt="32" labels="<O,12,U>" file="Speech012" channel="0" sequence="9" R_T1="1922.340" R_T2="1931.720" word_aux="h_t1+t2,h_conf">
45+
D,"right",,,:D,"and",,,:D,"it",,,:D,"is",,,:D,"it",,,:D,"is",,,:D,"important",,,:D,"for",,,:D,"us",,,:C,"(%partial)","",,:D,"to",,,:D,"understand",,,:D,"what",,,:D,"their",,,:D,"kpi",,,:D,"is",,,:D,"because",,,:D,"that",,,:D,"could",,,:D,"have",,,:D,"an",,,:D,"impact",,,:D,"on",,,:D,"how",,,:D,"we",,,:D,"what",,,:D,"we",,,:D,"want",,,:D,"to",,,:D,"measure",,,:D,"or",,,:D,"how",,,
46+
</PATH>
47+
</SPEAKER>
48+
<SPEAKER id="inter_segment_gap">
49+
<PATH id="(Inter_Segment_Gap-000)" word_cnt="1" file="Speech012" channel="0" sequence="19" R_T1="316.160" R_T2="600.490" word_aux="h_t1+t2,h_conf">
50+
I,,"so",316.160+316.450,7.070000
51+
</PATH>
52+
<PATH id="(Inter_Segment_Gap-001)" word_cnt="0" file="Speech012" channel="0" sequence="20" R_T1="602.150" R_T2="649.020" word_aux="h_t1+t2,h_conf">
53+
54+
</PATH>
55+
<PATH id="(Inter_Segment_Gap-002)" word_cnt="0" file="Speech012" channel="0" sequence="21" R_T1="658.270" R_T2="671.000" word_aux="h_t1+t2,h_conf">
56+
57+
</PATH>
58+
<PATH id="(Inter_Segment_Gap-003)" word_cnt="0" file="Speech012" channel="0" sequence="22" R_T1="675.900" R_T2="677.200" word_aux="h_t1+t2,h_conf">
59+
60+
</PATH>
61+
<PATH id="(Inter_Segment_Gap-004)" word_cnt="0" file="Speech012" channel="0" sequence="23" R_T1="684.510" R_T2="685.300" word_aux="h_t1+t2,h_conf">
62+
63+
</PATH>
64+
<PATH id="(Inter_Segment_Gap-005)" word_cnt="0" file="Speech012" channel="0" sequence="24" R_T1="686.920" R_T2="1617.620" word_aux="h_t1+t2,h_conf">
65+
66+
</PATH>
67+
<PATH id="(Inter_Segment_Gap-006)" word_cnt="0" file="Speech012" channel="0" sequence="25" R_T1="1619.950" R_T2="1874.480" word_aux="h_t1+t2,h_conf">
68+
69+
</PATH>
70+
<PATH id="(Inter_Segment_Gap-007)" word_cnt="0" file="Speech012" channel="0" sequence="26" R_T1="1875.730" R_T2="1876.300" word_aux="h_t1+t2,h_conf">
71+
72+
</PATH>
73+
<PATH id="(Inter_Segment_Gap-008)" word_cnt="0" file="Speech012" channel="0" sequence="27" R_T1="1880.310" R_T2="1880.920" word_aux="h_t1+t2,h_conf">
74+
75+
</PATH>
76+
<PATH id="(Inter_Segment_Gap-009)" word_cnt="0" file="Speech012" channel="0" sequence="28" R_T1="1887.420" R_T2="1922.340" word_aux="h_t1+t2,h_conf">
77+
78+
</PATH>
79+
</SPEAKER>
80+
</SYSTEM>

‎src/asclite/testfiles/isgbug-fail.ctm

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Speech012 0 316.160 0.290 SO 7.07
2+
Speech012 0 316.450 0.160 DO 7.07

‎src/asclite/testfiles/isgbug-work.ctm

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Speech012 0 316.160 0.290 SO 7.07

‎src/asclite/testfiles/isgbug.stm

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
;; Reference file for Speech012, generated 2019-01-22 08:32:44.069253
2+
;; CATEGORY "0" "" ""
3+
;; LABEL "O" "Overall" "The Complete Test Set"
4+
;; CATEGORY "1" "Corpus" ""
5+
;; LABEL "12" "Speech012" ""
6+
;; CATEGORY "2" "Gender" ""
7+
;; LABEL "F" "Female" "Female Participant"
8+
;; LABEL "M" "Male" "Male Participant"
9+
Speech012 0 sp002 0.00 0.00 <O,12,U> IGNORE_TIME_SEGMENT_IN_SCORING
10+
Speech012 0 sp002 600.49 602.15 <O,12,U> GOT YOUR MEETING REQUEST
11+
Speech012 0 sp002 649.02 658.27 <O,12,U> YEAH (%PARTIAL) I WOULD LOVE (%PARTIAL) TO GRAB SOME TIME WITH YOU (%HESITATION) SEPARATELY AND (%HESITATION) GET RAMPED UP AND ALSO MAYBE (%PARTIAL) GET MY HANDS ON SOME DATA
12+
Speech012 0 sp002 671.00 675.90 <O,12,U> WOULD YOU MIND INCLUDING PAVAN (%PARTIAL) HE MIGHT { HE HAD / HE WOULD } WOULD HAVE SOME (%PARTIAL) GOOD IDEAS TOO
13+
Speech012 0 sp002 677.20 684.51 <O,12,U> OUR NEW DATA SCIENTIST I WANT HIM TO (%PARTIAL) TO WORK THAT THOSE MUSCLES (%LAUGHTER)
14+
Speech012 0 sp002 685.30 686.92 <O,12,U> MUSCLES THANK YOU
15+
Speech012 0 sp002 1617.62 1619.95 <O,12,U> (%LAUGHTER) NO
16+
Speech012 0 sp002 1874.48 1875.73 <O,12,U> LI JUAN
17+
Speech012 0 sp002 1876.30 1880.31 <O,12,U> I SEE SOMETHING THAT SAYS KPI METRICS
18+
Speech012 0 sp002 1880.92 1887.42 <O,12,U> IS THERE (%HESITATION) OTHER KPI LISTED SOMEWHERE
19+
Speech012 0 sp002 1922.34 1931.72 <O,12,U> RIGHT AND { IT IS / IT HAS / IT WAS } { IT IS / IT HAS / IT WAS } IMPORTANT FOR US (%PARTIAL) TO UNDERSTAND WHAT THEIR KPI IS BECAUSE THAT COULD HAVE AN IMPACT ON HOW WE WHAT WE WANT TO MEASURE OR HOW

0 commit comments

Comments
 (0)
Please sign in to comment.