samples/models/dummy/config.json

@@ -3,8 +3,6 @@
   "tags": ["dummy"],
   "file": "model.py",
 
-  "output_type": "video/mp4",
-
   "inputs": {
     "file": {"type": "file"}
   }

samples/models/dummy/model.py

@@ -1,3 +1,4 @@
+import json
 import typing
 
 
@@ -7,5 +8,5 @@ def load(model) -> None:
 def unload(model) -> None:
     pass
 
-async def infer(model, file) -> typing.AsyncIterator[bytes]:
+def infer(model, file) -> typing.Iterator[bytes]:
-    yield await file.read()
+    yield json.dumps({"hello": "world!"}).encode("utf-8")

source/manager/ModelManager.py

@@ -1,4 +1,3 @@
-import asyncio
 import json
 import os
 import typing
@@ -11,11 +10,6 @@ from source import model, api
 
 
 class ModelManager:
-    """
-    The model manager
-    Load the list of models available, ensure that only one model is loaded at the same time.
-    """
-
     def __init__(self, application: api.Application, model_library: os.PathLike | str):
         self.application: api.Application = application
         self.model_library: Path = Path(model_library)
@@ -26,14 +20,9 @@ class ModelManager:
         self.models: dict[str, model.base.BaseModel] = {}
 
         # the currently loaded model
-        # TODO(Faraphel): load more than one model at a time ?
+        # TODO(Faraphel): load more than one model at a time ? require a way more complex manager to handle memory issue
-        #  would require a way more complex manager to handle memory issue
-        #  having two calculations at the same time might not be worth it either
         self.current_loaded_model: typing.Optional[model.base.BaseModel] = None
 
-        # lock to avoid concurrent inference and concurrent model loading and unloading
-        self.inference_lock = asyncio.Lock()
-
         @self.application.get("/models")
         async def get_models() -> list[str]:
             """

source/model/PythonModel.py

@@ -50,7 +50,7 @@ class PythonModel(base.BaseModel):
         parameters = utils.parameters.load(configuration.get("inputs", {}))
 
         # create an endpoint wrapping the inference inside a fastapi call
-        async def infer_api(**kwargs) -> fastapi.responses.StreamingResponse:
+        async def infer_api(**kwargs):
             # NOTE: fix an issue where it is not possible to give an UploadFile to a StreamingResponse
             # NOTE: perform a naive type(value).__name__ == "type_name" because fastapi do not use it own
             #   fastapi.UploadFile class, but instead the starlette UploadFile class that is more of an implementation
@@ -61,12 +61,8 @@ class PythonModel(base.BaseModel):
             }
 
             return fastapi.responses.StreamingResponse(
-                content=await self.infer(**kwargs),
+                content=self.infer(**kwargs),
                 media_type=self.output_type,
-                headers={
-                    # if the data is not text-like, mark it as an attachment to avoid display issue with Swagger UI
-                    "content-disposition": "inline" if utils.mimetypes.is_textlike(self.output_type) else "attachment"
-                }
             )
 
         infer_api.__signature__ = inspect.Signature(parameters=parameters)
@@ -77,12 +73,6 @@ class PythonModel(base.BaseModel):
             infer_api,
             methods=["POST"],
             tags=self.tags,
-            # summary=...,
-            # description=...,
-            response_class=fastapi.responses.StreamingResponse,
-            responses={
-                200: {"content": {self.output_type: {}}}
-            },
         )
 
     def _load(self) -> None:
@@ -91,5 +81,5 @@ class PythonModel(base.BaseModel):
     def _unload(self) -> None:
         return self.module.unload(self)
 
-    def _infer(self, **kwargs) -> typing.Iterator[bytes] | typing.Iterator[bytes]:
+    def _infer(self, **kwargs) -> typing.Iterator[bytes]:
         return self.module.infer(self, **kwargs)

source/model/base/BaseModel.py

@@ -106,13 +106,12 @@ class BaseModel(abc.ABC):
         Do not call manually, use `unload` instead.
         """
 
-    async def infer(self, **kwargs) -> typing.Iterator[bytes] | typing.AsyncIterator[bytes]:
+    def infer(self, **kwargs) -> typing.Iterator[bytes]:
         """
         Infer our payload through the model within the model manager
         :return: the response of the model
         """
 
-        async with self.manager.inference_lock:
         # make sure we are loaded before an inference
         self.load()
 
@@ -120,7 +119,7 @@ class BaseModel(abc.ABC):
         return self._infer(**kwargs)
 
     @abc.abstractmethod
-    def _infer(self, **kwargs) -> typing.Iterator[bytes] | typing.AsyncIterator[bytes]:
+    def _infer(self, **kwargs) -> typing.Iterator[bytes]:
         """
         Infer our payload through the model
         :return: the response of the model

source/utils/__init__.py

@@ -1,2 +1 @@
 from . import parameters
-from . import mimetypes

source/utils/mimetypes.py

@@ -1,21 +0,0 @@
-def is_textlike(mimetype: str) -> bool:
-    """
-    Determinate if a mimetype is considered as holding text
-    :param mimetype: the mimetype to check
-    :return: True if the mimetype represent text, False otherwise
-    """
-
-    # check the family of the mimetype
-    if mimetype.startswith("text/"):
-        return True
-
-    # check applications formats that are text formatted
-    if mimetype in [
-        "application/xml",
-        "application/json",
-        "application/javascript"
-    ]:
-        return True
-
-    # otherwise consider the file as non-text
-    return False