Skip to content

Commit dc95d21

Browse files
committed
benchmark: Use AsSpan() for MicrosoftMLTokenizer.
1 parent 6413a0a commit dc95d21

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

src/benchmarks/Tiktoken.Benchmarks/Benchmarks.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public async Task GlobalSetup()
4141

4242
[Benchmark]
4343
[BenchmarkCategory("Encode")]
44-
public IReadOnlyCollection<int> MicrosoftMLTokenizerV1_0_0_Encode() => _microsoftMlTiktoken.EncodeToIds(Data);
44+
public IReadOnlyCollection<int> MicrosoftMLTokenizerV1_0_0_Encode() => _microsoftMlTiktoken.EncodeToIds(Data.AsSpan());
4545

4646
[Benchmark]
4747
[BenchmarkCategory("Encode")]
@@ -62,7 +62,7 @@ public async Task GlobalSetup()
6262

6363
[Benchmark]
6464
[BenchmarkCategory("CountTokens")]
65-
public int MicrosoftMLTokenizerV1_0_0_() => _microsoftMlTiktoken.CountTokens(Data);
65+
public int MicrosoftMLTokenizerV1_0_0_() => _microsoftMlTiktoken.CountTokens(Data.AsSpan());
6666

6767
[Benchmark]
6868
[BenchmarkCategory("CountTokens")]

0 commit comments

Comments
 (0)