promptcache/cache_engine.py

[liyanboSustech]

class SchemaCache:
    schema: Schema
    cache_l1: Dict[int, TokenSequenceCache]
    cache_l2: Dict[Tuple[int, int], Tuple[TokenSequenceCache, TokenSequenceCache]]

    lm: LanguageModel

    def __init__(self, schema: Schema, lm: LanguageModel, batch_size: int = 1, target_device=None, no_cache=False):
        self.schema = schema
        self.lm = lm
        self.cache_l1 = dict()
        self.cache_l2 = dict()
        self.target_device = lm.device if target_device is None else target_device
        # no_cache 默认是False，那么if not no_cache就是True
        if not no_cache:
            self._process(batch_size)

why should the batchsize to be specified as 1 when initializing the SchemaCache？ thanks for your early reply❤ @forrestzhong