Getting "Torch not compiled with CUDA enabled error"

[dhirajkumarsinha27091982]

I have CPU based windows system and while running the below code I get the "Torch not compiled with CUDA enabled error".

Code:-
RAG = RAGMultiModalModel.from_pretrained("vidore/colpali")

Error:-

AssertionError Traceback (most recent call last)
Cell In[20], line 1
----> 1 RAG = RAGMultiModalModel.from_pretrained("vidore/colpali")

File ~\AppData\Roaming\Python\Python310\site-packages\byaldi\RAGModel.py:59, in RAGMultiModalModel.from_pretrained(cls, pretrained_model_name_or_path, index_root, device, verbose)
49 """Load a ColPali model from a pre-trained checkpoint.
50
51 Parameters:
(...)
56 cls (RAGMultiModalModel): The current instance of RAGMultiModalModel, with the model initialised.
57 """
58 instance = cls()
---> 59 instance.model = ColPaliModel.from_pretrained(
60 pretrained_model_name_or_path,
61 index_root=index_root,
62 device=device,
63 verbose=verbose,
64 )
65 return instance

File ~\AppData\Roaming\Python\Python310\site-packages\byaldi\colpali.py:209, in ColPaliModel.from_pretrained(cls, pretrained_model_name_or_path, n_gpu, verbose, device, index_root, **kwargs)
199 @classmethod
200 def from_pretrained(
201 cls,
...
312 raise AssertionError(
313 "libcudart functions unavailable. It looks like you have a broken build?"
314 )

AssertionError: Torch not compiled with CUDA enabled

[bNTyARH1NeSTX]

this worked for me, rag = RAGMultiModalModel.from_pretrained(
"vidore/colqwen2-v1.0",
index_root=index_root,
device="cuda:0"

[bNTyARH1NeSTX]

colpali just updated so that is why I think you are getting the error

[dhirajkumarsinha27091982]

this worked for me, rag = RAGMultiModalModel.from_pretrained( "vidore/colqwen2-v1.0", index_root=index_root, device="cuda:0"

Hmm still i got same error with above setting that you have mentioned however below setting worked for me:-
RAG = RAGMultiModalModel.from_pretrained("vidore/colqwen2-v1.0",device="cpu")
However I dont whether in this setting the ColVision model will be able to do the proper indexing of the Graphics PDF.

[aihacker111]

I'm break their code and fixed it

from pathlib import Path
from typing import Any, Dict, List, Optional, Union

from PIL import Image

from .colpali import ColPaliModel

from .objects import Result

try:
    from intergrations.langchain import LangChainRetriever
except ImportError:
    pass


class RAGMultiModalModel:
    """
    Wrapper class for a pretrained RAG multi-modal model, and all the associated utilities.
    Allows you to load a pretrained model from disk or from the hub, build or query an index.

    ## Usage

    Load a pre-trained checkpoint:

    python
    from byaldi import RAGMultiModalModel

    RAG = RAGMultiModalModel.from_pretrained("vidore/colpali-v1.2")
   

    Both methods will load a fully initialised instance of ColPali, which you can use to build and query indexes.

    python
    RAG.search("How many people live in France?")
   
    """

    model: Optional[ColPaliModel] = None

    @classmethod
    def from_pretrained(
        cls,
        pretrained_model_name_or_path: Union[str, Path],
        index_root: str = ".byaldi",
        device: str = "cuda",
        verbose: int = 1,
    ):
        """Load a ColPali model from a pre-trained checkpoint.

        Parameters:
            pretrained_model_name_or_path (str): Local path or huggingface model name.
            device (str): The device to load the model on. Default is "cuda".

        Returns:
            cls (RAGMultiModalModel): The current instance of RAGMultiModalModel, with the model initialised.
        """
        instance = cls()
        instance.model = ColPaliModel.from_pretrained(
            pretrained_model_name_or_path,
            index_root=index_root,
            device=device,
            verbose=verbose,
        )
        return instance

    @classmethod
    def from_index(
        cls,
        index_path: Union[str, Path],
        index_root: str = ".byaldi",
        device: str = "cuda",
        verbose: int = 1,
    ):
        """Load an Index and the associated ColPali model from an existing document index.

        Parameters:
            index_path (Union[str, Path]): Path to the index.
            device (str): The device to load the model on. Default is "cuda".

        Returns:
            cls (RAGMultiModalModel): The current instance of RAGMultiModalModel, with the model and index initialised.
        """
        instance = cls()
        index_path = Path(index_path)
        instance.model = ColPaliModel.from_index(
            index_path, index_root=index_root, device=device, verbose=verbose
        )

        return instance

    def index(
        self,
        input_path: Union[str, Path],
        index_name: Optional[str] = None,
        doc_ids: Optional[int] = None,
        store_collection_with_index: bool = False,
        overwrite: bool = False,
        metadata: Optional[
            Union[
                Dict[Union[str, int], Dict[str, Union[str, int]]],
                List[Dict[str, Union[str, int]]],
            ]
        ] = None,
        max_image_width: Optional[int] = None,
        max_image_height: Optional[int] = None,
        **kwargs,
    ):
        """Build an index from input documents.

        Parameters:
            input_path (Union[str, Path]): Path to the input documents.
            index_name (Optional[str]): The name of the index that will be built.
            doc_ids (Optional[List[Union[str, int]]]): List of document IDs.
            store_collection_with_index (bool): Whether to store the collection with the index.
            overwrite (bool): Whether to overwrite an existing index with the same name.
            metadata (Optional[Union[Dict[Union[str, int], Dict[str, Union[str, int]]], List[Dict[str, Union[str, int]]]]]):
                Metadata for the documents. Can be a dictionary mapping doc_ids to metadata dictionaries,
                or a list of metadata dictionaries (one for each document).

        Returns:
            None
        """
        return self.model.index(
            input_path,
            index_name,
            doc_ids,
            store_collection_with_index,
            overwrite=overwrite,
            metadata=metadata,
            max_image_width=max_image_width,
            max_image_height=max_image_height,
            **kwargs,
        )

    def add_to_index(
        self,
        input_item: Union[str, Path, Image.Image],
        store_collection_with_index: bool,
        doc_id: Optional[int] = None,
        metadata: Optional[Dict[str, Union[str, int]]] = None,
    ):
        """Add an item to an existing index.

        Parameters:
            input_item (Union[str, Path, Image.Image]): The item to add to the index.
            store_collection_with_index (bool): Whether to store the collection with the index.
            doc_id (Union[str, int]): The document ID for the item being added.
            metadata (Optional[Dict[str, Union[str, int]]]): Metadata for the document being added.

        Returns:
            None
        """
        return self.model.add_to_index(
            input_item, store_collection_with_index, doc_id, metadata=metadata
        )

    def search(
        self,
        query: Union[str, List[str]],
        k: int = 10,
        filter_metadata: Optional[Dict[str,str]] = None,
        return_base64_results: Optional[bool] = None,
    ) -> Union[List[Result], List[List[Result]]]:
        """Query an index.

        Parameters:
            query (Union[str, List[str]]): The query or queries to search for.
            k (int): The number of results to return. Default is 10.
            return_base64_results (Optional[bool]): Whether to return base64-encoded image results.

        Returns:
            Union[List[Result], List[List[Result]]]: A list of Result objects or a list of lists of Result objects.
        """
        return self.model.search(query, k, filter_metadata, return_base64_results)

    def get_doc_ids_to_file_names(self):
        return self.model.get_doc_ids_to_file_names()

    def as_langchain_retriever(self, **kwargs: Any):
        return LangChainRetriever(model=self, kwargs=kwargs)

You can replace it and run again on GPU