stream.py

# Copyright    2022  Xiaomi Corp.        (authors: Fangjun Kuang,
#                                                  Zengwei Yao)
#
# See ../../../../LICENSE for clarification regarding multiple authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math
from typing import List, Optional, Tuple

import k2
import torch
from beam_search import Hypothesis, HypothesisList

from icefall.utils import AttributeDict


class Stream(object):
    def __init__(
        self,
        params: AttributeDict,
        cut_id: str,
        decoding_graph: Optional[k2.Fsa] = None,
        device: torch.device = torch.device("cpu"),
        LOG_EPS: float = math.log(1e-10),
    ) -> None:
        """
        Args:
          params:
            It's the return value of :func:`get_params`.
          cut_id:
            The cut id of the current stream.
          decoding_graph:
            The decoding graph. Can be either a `k2.trivial_graph` or HLG, Used
            only when --decoding_method is fast_beam_search.
          device:
            The device to run this stream.
          LOG_EPS:
            A float value used for padding.
        """
        self.LOG_EPS = LOG_EPS
        self.cut_id = cut_id

        # Containing attention caches and convolution caches
        self.states: Optional[Tuple[torch.Tensor, torch.Tensor]] = None

        # It uses different attributes for different decoding methods.
        self.context_size = params.context_size
        self.decoding_method = params.decoding_method
        if params.decoding_method == "greedy_search":
            self.hyp = [params.blank_id] * params.context_size
        elif params.decoding_method == "modified_beam_search":
            self.hyps = HypothesisList()
            self.hyps.add(
                Hypothesis(
                    ys=[params.blank_id] * params.context_size,
                    log_prob=torch.zeros(1, dtype=torch.float32, device=device),
                )
            )
        elif params.decoding_method == "fast_beam_search":
            # feature_len is needed to get partial results.
            # The rnnt_decoding_stream for fast_beam_search.
            self.rnnt_decoding_stream: k2.RnntDecodingStream = (
                k2.RnntDecodingStream(decoding_graph)
            )
            self.hyp: Optional[List[int]] = None
        else:
            raise ValueError(
                f"Unsupported decoding method: {params.decoding_method}"
            )

        self.ground_truth: str = ""

        self.feature: Optional[torch.Tensor] = None
        # Make sure all feature frames can be used.
        # We aim to obtain 1 frame after subsampling.
        self.chunk_length = params.subsampling_factor
        self.pad_length = 5
        self.num_frames = 0
        self.num_processed_frames = 0

        # After all feature frames are processed, we set this flag to True
        self._done = False

    def set_feature(self, feature: torch.Tensor) -> None:
        assert feature.dim() == 2, feature.dim()
        # tail padding here to alleviate the tail deletion problem
        num_tail_padded_frames = 35
        self.num_frames = feature.size(0) + num_tail_padded_frames
        self.feature = torch.nn.functional.pad(
            feature,
            (0, 0, 0, self.pad_length + num_tail_padded_frames),
            mode="constant",
            value=self.LOG_EPS,
        )

    def get_feature_chunk(self) -> torch.Tensor:
        """Get a chunk of feature frames.

        Returns:
          A tensor of shape (ret_length, feature_dim).
        """
        update_length = min(
            self.num_frames - self.num_processed_frames, self.chunk_length
        )
        ret_length = update_length + self.pad_length

        ret_feature = self.feature[
            self.num_processed_frames : self.num_processed_frames + ret_length
        ]
        # Cut off used frames.
        # self.feature = self.feature[update_length:]

        self.num_processed_frames += update_length
        if self.num_processed_frames >= self.num_frames:
            self._done = True

        return ret_feature

    @property
    def id(self) -> str:
        return self.cut_id

    @property
    def done(self) -> bool:
        """Return True if all feature frames are processed."""
        return self._done

    def decoding_result(self) -> List[int]:
        """Obtain current decoding result."""
        if self.decoding_method == "greedy_search":
            return self.hyp[self.context_size :]
        elif self.decoding_method == "modified_beam_search":
            best_hyp = self.hyps.get_most_probable(length_norm=True)
            return best_hyp.ys[self.context_size :]
        else:
            assert self.decoding_method == "fast_beam_search"
            return self.hyp