http_server first commit

http_submission/Dockerfile

@@ -0,0 +1,23 @@
+# Specify a base image depending on the project.
+FROM bitnami/python:3.8
+# For more complex examples, might need to use a different base image.
+# FROM pytorch/pytorch:1.9.1-cuda11.1-cudnn8-runtime
+
+WORKDIR /app
+
+ENV HTTP_PORT=4000
+
+RUN apt-get update \
+    && apt-get -y install gcc
+
+COPY ./requirements.txt ./
+RUN python -m pip install -U pip \
+    && python -m pip install -r requirements.txt 
+
+COPY . ./
+
+# This is needed for Singularity builds.
+EXPOSE $HTTP_PORT
+
+# The entrypoint for a container,
+CMD ["gunicorn", "-w", "1", "-b", "0.0.0.0:4000", "--pythonpath", ".", "model_server:app"]

http_submission/README.md

@@ -0,0 +1,27 @@
+# Submission
+TODO: Add a description of the submission process here.
+
+
+
+## Launching the submission container
+TODO: Create a docker-compose file
+```bash
+cd ./http_submission
+docker build -t sample_pysaliency .
+```
+
+```bash
+docker run --name sample_pysaliency -dp 4000:4000 sample_pysaliency
+```
+The above command will launch a container named `sample_pysaliency` and expose the port `4000` to the host machine. The container will be running in the background. 
+
+To test the model server, run the sample_evaluation script (Make sure to have the `pysaliency` package installed):
+```bash
+python ./http_evaluation/sample_evaluation.py
+```
+
+
+To delete the container, run the following command:
+```bash
+docker stop sample_pysaliency && docker rm sample_pysaliency
+```

http_submission/model_server.py

@@ -0,0 +1,46 @@
+from flask import Flask, request, jsonify
+import numpy as np
+import json
+from PIL import Image
+from io import BytesIO
+# import pickle
+
+# Import your model here
+from sample_submission import MySimpleScanpathModel
+
+app = Flask("saliency-model-server")
+app.logger.setLevel("DEBUG")
+
+# # TODO - replace this with your model
+model = MySimpleScanpathModel()
+
+
+@app.route('/conditional_log_density', methods=['POST'])
+def conditional_log_density():
+    data = json.loads(request.form['json_data'])
+    x_hist = np.array(data['x_hist'])
+    y_hist = np.array(data['y_hist'])
+    t_hist = np.array(data['t_hist'])
+    attributes = data.get('attributes', {})
+
+    image_bytes = request.files['stimulus'].read()
+    image = Image.open(BytesIO(image_bytes))
+    stimulus = np.array(image)
+
+    log_density = model.conditional_log_density(stimulus, x_hist, y_hist, t_hist, attributes)
+    return jsonify({'log_density': log_density.tolist()})
+
+
+@app.route('/type', methods=['GET'])
+def type():
+    type = "ScanpathModel"
+    version = "v1.0.0"
+    return jsonify({'type': type, 'version': version})
+
+
+def main():
+    app.run(host="localhost", port="4000", debug="True", threaded=True)
+
+
+if __name__ == "__main__":
+    main()

http_submission/requirements.txt

@@ -0,0 +1,6 @@
+cython
+flask
+gunicorn
+numpy
+
+# Add additional dependencies here

http_submission/sample_evaluation.py

@@ -0,0 +1,89 @@
+import numpy as np
+import pickle
+import requests
+import sys
+from sample_submission import MySimpleScanpathModel
+from PIL import Image
+from io import BytesIO
+import json
+import matplotlib.pyplot as plt
+from pysaliency.plotting import plot_scanpath
+sys.path.insert(0, '..')
+import pysaliency
+
+class HTTPScanpathModel(MySimpleScanpathModel):
+    def __init__(self, url):
+        self.url = url
+        self.log_density_url = url + "/conditional_log_density"
+        self.type_url = url + "/type"
+
+
+
+
+    def conditional_log_density(self, stimulus, x_hist, y_hist, t_hist, attributes=None, out=None):
+
+        # build request
+        pil_image = Image.fromarray(stimulus)
+        image_bytes = BytesIO()
+        pil_image.save(image_bytes, format='png')
+
+        def _convert_attribute(attribute):
+            if isinstance(attribute, np.ndarray):
+                return attribute.tolist()
+            return attribute
+
+        json_data = {
+            "x_hist": list(x_hist),
+            "y_hist": list(y_hist),
+            "t_hist": list(t_hist),
+            "attributes": {key: _convert_attribute(value) for key, value in (attributes or {}).items()}
+        }
+
+        # send request
+        response = requests.post(f"{self.log_density_url}", data={'json_data': json.dumps(json_data)}, files={'stimulus': image_bytes.getvalue()})
+
+        # parse response
+        if response.status_code != 200:
+            raise ValueError(f"Server returned status code {response.status_code}")
+
+        return np.array(response.json()['log_density'])
+
+    def type(self):
+        response = requests.get(f"{self.type_url}")
+        return np.array(response.json())
+
+
+if __name__ == "__main__":
+    http_model = HTTPScanpathModel("http://localhost:4000")
+
+    # get MIT1003 dataset
+    stimuli, fixations = pysaliency.get_mit1003(location='pysaliency_datasets')
+    fixation_index = 32185
+
+    # get server response for one stimulus
+    server_density = http_model.conditional_log_density(
+        stimulus=stimuli.stimuli[fixations.n[fixation_index]], 
+        x_hist=fixations.x_hist[fixation_index], 
+        y_hist=fixations.y_hist[fixation_index], 
+        t_hist=fixations.t_hist[fixation_index]
+    )
+    # get server version
+    server_version = http_model.type()
+    print(server_version)
+
+
+    # TODO: delete plotting part
+    # plot server response, only for testing
+
+    fig, axs = plt.subplots(1, 2, figsize=(12, 6))
+    axs[0].set_axis_off()
+    axs[1].set_axis_off()
+
+    axs[0].imshow(stimuli.stimuli[fixations.n[fixation_index]])
+    plot_scanpath(stimuli, fixations, fixation_index, visualize_next_saccade=True, ax=axs[0])
+    axs[0].set_title("Image")
+
+    axs[1].imshow(server_density)
+
+    axs[1].set_title("http_model_log_density")
+    fig.savefig("test.png")

http_submission/sample_submission.py

@@ -0,0 +1,79 @@
+import numpy as np
+import sys
+from typing import Union
+from scipy.ndimage import gaussian_filter
+sys.path.insert(0, '..')
+import pysaliency
+
+
+class LocalContrastModel(pysaliency.Model):
+    def __init__(self, bandwidth=0.05, **kwargs):
+        super().__init__(**kwargs)
+        self.bandwidth = bandwidth
+
+    def _log_density(self, stimulus: Union[pysaliency.datasets.Stimulus, np.ndarray]):
+
+        # _log_density can either take pysaliency Stimulus objects, or, for convenience, simply numpy arrays
+        # `as_stimulus` ensures that we have a Stimulus object
+        stimulus_object = pysaliency.datasets.as_stimulus(stimulus)
+
+        # grayscale image
+        gray_stimulus = np.mean(stimulus_object.stimulus_data, axis=2)
+
+        # size contains the height and width of the image, but not potential color channels
+        height, width = stimulus_object.size
+
+        # define kernel size based on image size
+        kernel_size = np.round(self.bandwidth * max(width, height)).astype(int)
+        sigma = (kernel_size - 1) / 6
+
+        # apply Gausian blur and calculate squared difference between blurred and original image
+        blurred_stimulus = gaussian_filter(gray_stimulus, sigma)
+
+        prediction = gaussian_filter((gray_stimulus - blurred_stimulus)**2, sigma)
+
+        # normalize to [1, 255]
+        prediction = (254 * (prediction / prediction.max())).astype(int) + 1
+
+        density = prediction / prediction.sum()
+
+        return np.log(density)
+
+class MySimpleScanpathModel(pysaliency.ScanpathModel):
+    def __init__(self, spatial_model_bandwidth: float=0.05, saccade_width: float=0.1):
+        self.spatial_model_bandwidth = spatial_model_bandwidth
+        self.saccade_width = saccade_width
+        self.spatial_model = LocalContrastModel(spatial_model_bandwidth)
+
+
+    def conditional_log_density(self, stimulus, x_hist, y_hist, t_hist, attributes=None, out=None,):
+        stimulus_object = pysaliency.datasets.as_stimulus(stimulus)
+
+        # size contains the height and width of the image, but not potential color channels
+        height, width = stimulus_object.size
+
+        spatial_prior_log_density = self.spatial_model.log_density(stimulus)
+        spatial_prior_density = np.exp(spatial_prior_log_density)
+
+        # compute saccade bias
+        last_x = x_hist[-1]
+        last_y = y_hist[-1]
+
+        xs = np.arange(width, dtype=float)
+        ys = np.arange(height, dtype=float)
+        XS, YS = np.meshgrid(xs, ys)
+
+        XS -= last_x
+        YS -= last_y
+
+        # compute prior
+        max_size = max(width, height)
+        actual_kernel_size = self.saccade_width * max_size
+
+        saccade_bias = np.exp(-0.5 * (XS ** 2 + YS ** 2) / actual_kernel_size ** 2)
+
+        prediction = spatial_prior_density * saccade_bias
+
+        density = prediction / prediction.sum()
+        return np.log(density)
+