from abc import ABC, abstractmethod
from typing import Dict, Any, List, Union
import torch

class BaseTokenizer(ABC):
    
    @abstractmethod
    def encode(self, text: str) -> List[int]:
        pass
    
    @abstractmethod
    def decode(self, ids: List[int]) -> List[str]:
        pass
    
    @abstractmethod
    def __call__(
        self,
        text: Union[str, List[str]],
        max_length: int = None,
        padding: bool = False,
        truncation: bool = False,
        return_tensors: str = None
    ) -> Dict[str, torch.Tensor]:
        pass
    
    @property
    @abstractmethod
    def vocab_size(self) -> int:
        pass
    
    @property
    @abstractmethod
    def pad_token_id(self) -> int:
        pass
    
    @property
    @abstractmethod
    def eos_token_id(self) -> int:
        pass 