added flask app

.example.env

@@ -1,3 +1,6 @@
+OPENAI_API_KEY=your-openai-key
+OPENAI_MODEL_VERSION=gpt-3.5-turbo-0613
+
 COMPOSE_PROJECT_NAME=dockerosm
 # Change the following variable if you want to merge multiple compose files
 # docker-compose will automatically merged docker-compose.yml and docker-compose.override.yml if exists.

.gitignore

@@ -19,3 +19,5 @@ settings/clip/clip.shx
 
 .venv
 .env
+
+*.webm

docker-compose.yml

@@ -8,6 +8,16 @@ volumes:
   cache:
 
 services:
+  cartobot:
+    build: ./flask_app
+    container_name: cartobot
+    ports:
+      - "5000:5000"
+    volumes:
+      - ./flask_app/templates:/app/templates
+      - ./flask_app/uploads/audio:/app/uploads/audio
+    env_file: .env
+    restart: always
   db:
     image: kartoza/postgis:${POSTGRES_VERSION}
     environment:

flask_app/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.9-slim
+
+ENV PYTHONUNBUFFERED=1
+
+# Set working directory
+WORKDIR /app
+
+# Copy requirements and install
+COPY requirements.txt /app/
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application
+COPY . /app/
+
+# Expose the port the app runs on
+EXPOSE 5000
+
+CMD ["python", "app.py"]

flask_app/agents/function_descriptions/map_info_function_descriptions.py

@@ -0,0 +1,18 @@
+select_layer_name = {
+    "name": "select_layer_name",
+    "description": "Gets a layer name from the text of a given task related to selecting layers on a map.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "layer_name": {
+                "type": "string",
+                "description": "The name of the layer.",
+            },
+        },
+        "required": ["layer_name"],
+    },
+}
+
+map_info_function_descriptions = [
+    select_layer_name,
+]

flask_app/agents/function_descriptions/navigation_function_descriptions.py

@@ -0,0 +1,74 @@
+go_to_location = {
+    "name": "go_to_location",
+    "description": "Go to a given location.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "latitude": {
+                "type": "number",
+                "description": "The latitude to go to.",
+            },
+            "longitude": {
+                "type": "number",
+                "description": "The longitude to go to.",
+            },
+        },
+        "required": ["latitude", "longitude"],
+    },
+}
+
+pan_in_direction = {
+    "name": "pan_in_direction",
+    "description": "Pan in a given direction and distance in kilometers.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "direction": {
+                "type": "string",
+                "description": "The direction to pan in. One of 'north', 'south', 'east', 'west', 'northwest', 'northeast', 'southwest', 'southeast'.",
+            },
+            "distance_in_kilometers": {
+                "type": "number",
+                "description": "The distance to pan in kilometers. If not provided, defaults to 1.",
+            },
+        },
+        "required": ["direction"],
+    },
+}
+
+zoom_in = {
+    "name": "zoom_in",
+    "description": "Zoom in by a given number of zoom levels.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "zoom_levels": {
+                "type": "number",
+                "description": "The number of zoom levels to zoom in. If not provided, defaults to 1.",
+            },
+        },
+    },
+    "required": ["zoom_levels"],
+}
+
+zoom_out = {
+    "name": "zoom_out",
+    "description": "Zoom out by a given number of zoom levels.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "zoom_levels": {
+                "type": "number",
+                "description": "The number of zoom levels to zoom out. If not provided, defaults to 1.",
+            },
+        },
+    },
+    "required": ["zoom_levels"],
+}
+
+navigation_function_descriptions = [
+    go_to_location,
+    pan_in_direction,
+    zoom_in,
+    zoom_out,
+]

flask_app/agents/function_descriptions/style_function_descriptions.py

@@ -0,0 +1,83 @@
+
+set_color = {
+    "name": "set_color",
+    "description": "Set the maplibre paint property color of a layer.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "layer_name": {
+                "type": "string",
+                "description": "The name of the layer.",
+            },
+            "color": {
+                "type": "string", 
+                "description": "The color to set in hex format.",
+            },
+        },
+        "required": ["layer_name", "color"],
+    },
+}
+
+set_opacity = {
+    "name": "set_opacity",
+    "description": "Set the maplibre paint property opacity of a layer.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "layer_name": {
+                "type": "string",
+                "description": "The name of the layer.",
+            },
+            "opacity": {
+                "type": "number",
+                "description": "The opacity to set between 0 and 1.",
+            },
+        },
+        "required": ["layer_name", "opacity"],
+    },
+}
+
+set_width = {
+    "name": "set_width",
+    "description": "Set the maplibre paint property width.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "layer_name": {
+                "type": "string",
+                "description": "The name of the layer to get the paint property for.",
+            },
+            "width": {
+                "type": "number",
+                "description": "The width to set.",
+            },
+        },
+        "required": ["layer_name", "width"],
+    },
+}
+
+set_visibility = {
+    "name": "set_visibility",
+    "description": "Set the visibility of a layer (turning it on or off).",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "layer_name": {
+                "type": "string",
+                "description": "The name of the layer to get the layout property for.",
+            },
+            "visibility": {
+                "type": "string",
+                "description": "Either 'visible' or 'none'. Set to 'none' to hide the layer.",
+            },
+        },
+        "required": ["layer_name", "visible"],
+    },
+}
+
+style_function_descriptions = [
+    set_color,
+    set_opacity,
+    set_width,
+    set_visibility,
+]

flask_app/agents/map_info_agent.py

@@ -0,0 +1,125 @@
+import openai
+import json
+from .function_descriptions.map_info_function_descriptions import map_info_function_descriptions
+import logging
+
+logger = logging.getLogger(__name__)
+
+class MapInfoAgent:
+    """An agent that has function descriptions dealing with information about at GIS map and it's data (e.g. a MapLibre map)."""
+    
+    def select_layer_name(self, layer_name):
+        return {"name": "select_layer_name", "layer_name": layer_name}
+
+    def __init__(self, model_version="gpt-3.5-turbo-0613", layers=[]):
+        self.model_version = model_version
+        self.function_descriptions = map_info_function_descriptions
+        self.messages = [
+            {
+                "role": "system",
+                "content": """You are a helpful assistant that selects layers on a GIS map, e.g. a MapLibre map.
+                When asked to select, choose, or work on a layer, you will call the appropriate function.
+                Examples iclude:
+                'Select the layer named "Buildings"'
+                'Select the Roads layer'
+                'select contours'
+                'Select water-level-2'
+                'Work on highways'
+                'Work with Forests'
+                'Choose the Landuse layer'
+                'Choose the layer named Woods'
+                """,
+            },
+        ]
+        self.available_functions = {
+            "select_layer_name": self.select_layer_name,
+        }
+
+    def listen(self, message, layer_names=[]):
+        logger.info(f"In MapInfoAgent.listen()...message is: {message}")
+        """Listen to a message from the user."""
+        #map_context = f"The following layers are in the map: {layer_names}"
+        #remove the last item in self.messages
+        if len(self.messages) > 1:
+            self.messages.pop()
+        self.messages.append({
+            "role": "user",
+            "content": message,
+        })
+        # self.messages.append({
+        #     "role": "user",
+        #     "content": map_context,
+        # })
+        logger.info(f"MapInfoAgent self.messages: {self.messages}")
+        # this will be the function gpt will call if it 
+        # determines that the user wants to call a function
+        function_response = None
+
+        try:
+            response = openai.ChatCompletion.create(
+                model=self.model_version,
+                messages=self.messages,
+                functions=self.function_descriptions,
+                function_call="auto",
+                temperature=0.1,
+                max_tokens=256,
+                top_p=1,
+                frequency_penalty=0,
+                presence_penalty=0
+            )
+
+            response_message = response["choices"][0]["message"]
+
+            logger.info(f"First response from OpenAI in MapInfoAgent: {response_message}")
+
+            if response_message.get("function_call"):
+                function_name = response_message["function_call"]["name"]
+                logger.info(f"Type of function_name: {type(function_name)}")
+                function_name = function_name.strip()
+                logger.info(f"Function name: {function_name}")
+                function_to_call = self.available_functions[function_name]
+                logger.info(f"Function to call: {function_to_call}")
+                function_args = json.loads(response_message["function_call"]["arguments"])
+                logger.info(f"Function args: {function_args}")
+                # determine the function to call
+                function_response = function_to_call(**function_args)
+                self.messages.append(response_message)
+                self.messages.append({
+                    "role": "function",
+                    "name": function_name,
+                    "content": function_response,
+                })
+                logger.info(f"Function response: {function_response}")
+
+                # if the function name is select_layer_name, we need OpenAI to select the proper layer name instead of whatever the user said.
+                # They may have used lower case or a descriptive designation instead of the actual layer name. We hope OpenAI can figure out
+                # what they meant.
+                if function_name == "select_layer_name":
+                    logger.info(f"Sending layer name retrieval request to OpenAI...")
+                    prompt = f"Please select a layer name from the following list that is closest to the text '{function_response['layer_name']}': {str(layer_names)}\n Only state the layer name in your response."
+                    logger.info(f"Prompt to OpenAI: {prompt}")
+                    messages = [
+                        {
+                            "role": "user",
+                            "content": prompt,
+                        },
+                    ]
+                    second_response = openai.ChatCompletion.create(
+                        model=self.model_version,
+                        messages=messages,
+                    )
+                    logger.info(f"Second response from OpenAI in MapInfoAgent: {second_response}")
+                    second_response_message = second_response["choices"][0]["message"]["content"]
+                    logger.info(f"Second response message from OpenAI in MapInfoAgent: {second_response_message}")
+                    logger.info(f"Function Response bofore setting the layer name: {function_response}")
+                    function_response['layer_name'] = second_response_message
+                    logger.info(f"Function response after call to select a layername: {function_response}")
+                return {"response": function_response}
+            elif response_message.get("content"):
+                return {"response": response_message["content"]}
+            else:
+                return {"response": "I'm sorry, I don't understand."}
+        
+        except Exception as e:
+            return {"error": "Failed to get response from OpenAI in NavigationAgent: " + str(e)}, 500
+        return {"response": function_response}

flask_app/agents/marshall_agent.py

@@ -0,0 +1,94 @@
+import json
+import openai
+import logging
+
+logger = logging.getLogger(__name__)
+
+class MarshallAgent:
+    """A Marshall agent that has function descriptions for choosing the appropriate agent for a specified task."""
+    def __init__(self, model_version="gpt-3.5-turbo-0613"):
+        self.model_version = model_version
+
+        #we only need one function description for this agent
+        function_description = {
+            "name": "choose_agent",
+            "description": """Chooses an appropriate agent for a given task.""",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "agent_name": {
+                        "type": "string",
+                        "description": "The name of the agent to choose. One of 'NavigationAgent', 'StyleAgent', 'MapInfoAgent'.",
+                    },
+                },
+                "required": ["agent_name"],
+            },
+        }
+        
+        self.function_descriptions = [function_description]
+
+        self.messages = [
+            {
+                "role": "system",
+                "content": """You are a helpful assistant that decides which agent to use for a specified task. For tasks that ask to change
+                the style of a map, such as opacity, color, or line width, you will use the StyleAgent. For tasks that ask to manipulate layers on the map,
+                such as reordering, turning on and off, or getting the name of a layer, you will use the MapInfoAgent. For tasks that ask where something is, or to navigate
+                the map, such as panning, zooming, or flying to a location, you will use the NavigationAgent. If you can't find the appropriate agent, say that you didn't 
+                understand and ask for a more specific description of the task.""",
+            },
+        ]
+        self.logger = logging.getLogger(__name__)
+
+    def choose_agent(self, agent_name):
+        return {"name": "choose_agent", "agent_name": agent_name}
+
+    def listen(self, message):
+        self.logger.info(f"In MarshallAgent.listen()...message is: {message}")
+        """Listen to a message from the user."""
+        
+        # Remove the last item in self.messages. Our agent has no memory
+        if len(self.messages) > 1:
+            self.messages.pop()
+
+        self.messages.append({
+            "role": "user",
+            "content": message,
+        })
+        
+        # this will be the function gpt will choose if it 
+        # determines that the user wants to call a function
+        function_response = None
+
+        try:
+            response = openai.ChatCompletion.create(
+                model=self.model_version,
+                messages=self.messages,
+                functions=self.function_descriptions,
+                function_call={"name": "choose_agent"},
+                temperature=0,
+                max_tokens=256,
+                top_p=1,
+                frequency_penalty=0,
+                presence_penalty=0
+            )
+            response_message = response["choices"][0]["message"]
+
+            self.logger.info(f"Response from OpenAI in MarshallAgent: {response_message}")
+
+            if response_message.get("function_call"):
+                function_args = json.loads(response_message["function_call"]["arguments"])
+                self.logger.info(f"Function args: {function_args}")
+
+                # call choose agent
+                function_response = self.choose_agent(**function_args)
+                self.logger.info(f"Function response: {function_response}")
+
+                return {"response": function_response}
+            elif response_message.get("content"):
+                return {"response": response_message["content"]}
+            else:
+                return {"response": "I'm sorry, I don't understand."}
+        
+        except Exception as e:
+            return {"error": "Failed to get response from OpenAI in MarshallAgent: " + str(e)}, 500
+        return {"response": function_response}

flask_app/agents/navigation_agent.py

@@ -0,0 +1,96 @@
+import logging
+from .function_descriptions.navigation_function_descriptions import navigation_function_descriptions
+import openai
+import json
+
+logger = logging.getLogger(__name__)
+
+class NavigationAgent:
+    """A navigation agent that uses function calling for navigating the map."""
+    
+    def go_to_location(self, latitude, longitude):
+        return {"name": "go_to_location", "latitude": latitude, "longitude": longitude}
+
+    def pan_in_direction(self, direction, distance_in_kilometers=1):
+        return {"name": "pan_in_direction", "direction": direction, "distance_in_kilometers": distance_in_kilometers}
+
+    def zoom_in(self, zoom_levels=1):
+        return {"name": "zoom_in", "zoom_levels": zoom_levels}
+    
+    def zoom_out(self, zoom_levels=1):
+        return {"name": "zoom_out", "zoom_levels": zoom_levels}
+
+    def __init__(self, model_version="gpt-3.5-turbo-0613"):
+        self.model_version = model_version
+        self.function_descriptions = navigation_function_descriptions
+        self.messages = [
+            {
+                "role": "system",
+                "content": """You are a helpful assistant in navigating a maplibre map. When tasked to do something, you will call the appropriate function to navigate the map.
+
+                Examples tasks to go to a location include: 'Go to Tokyo', 'Where is the Eiffel Tower?', 'Navigate to New York City'
+
+                Examples tasks to pan in a direction include: 'Pan north 3 km', 'Pan ne 1 kilometer', 'Pan southwest', 'Pan east 2 kilometers', 'Pan south 5 kilometers', 'Pan west 1 kilometer', 'Pan northwest 2 kilometers', 'Pan southeast 3 kilometers'
+
+                Examples tasks to zoom in or zoom out include: 'Zoom in 2 zoom levels.', 'Zoom out 3', 'zoom out', 'zoom in', 'move closer', 'get closer', 'move further away', 'get further away', 'back out', 'closer' """,
+            },
+        ]
+        self.available_functions = {
+            "go_to_location": self.go_to_location,
+            "pan_in_direction": self.pan_in_direction,
+            "zoom_in": self.zoom_in,
+            "zoom_out": self.zoom_out,
+        }
+
+    def listen(self, message):
+        logging.info(f"In NavigationAgent.listen()...message is: {message}")
+        """Listen to a message from the user."""
+        #remove the last item in self.messages
+        if len(self.messages) > 1:
+            self.messages.pop()
+        self.messages.append({
+            "role": "user",
+            "content": message,
+        })
+        
+        # this will be the function gpt will call if it 
+        # determines that the user wants to call a function
+        function_response = None
+
+        try:
+            response = openai.ChatCompletion.create(
+                model=self.model_version,
+                messages=self.messages,
+                functions=self.function_descriptions,
+                function_call="auto",
+                temperature=0.1,
+                max_tokens=256,
+                top_p=1,
+                frequency_penalty=0,
+                presence_penalty=0
+            )
+
+            response_message = response["choices"][0]["message"]
+
+            logging.info(f"Response from OpenAI in NavigationAgent: {response_message}")
+
+            if response_message.get("function_call"):
+                function_name = response_message["function_call"]["name"]
+                logging.info(f"Function name: {function_name}")
+                function_to_call = self.available_functions[function_name]
+                logging.info(f"Function to call: {function_to_call}")
+                function_args = json.loads(response_message["function_call"]["arguments"])
+                logging.info(f"Function args: {function_args}")
+                # determine the function to call
+                function_response = function_to_call(**function_args)
+                logging.info(f"Function response: {function_response}")
+
+                return {"response": function_response}
+            elif response_message.get("content"):
+                return {"response": response_message["content"]}
+            else:
+                return {"response": "I'm sorry, I don't understand."}
+        
+        except Exception as e:
+            return {"error": "Failed to get response from OpenAI in NavigationAgent: " + str(e)}, 500
+        return {"response": function_response}

flask_app/agents/style_agent.py

@@ -0,0 +1,96 @@
+import logging
+from .function_descriptions.style_function_descriptions import style_function_descriptions
+import json
+import openai
+
+logger = logging.getLogger(__name__)
+
+class StyleAgent:
+    """A styling agent that has function descriptions for styling the map."""
+    def set_color(self, layer_name, color):
+        return {"name": "set_color", "layer_name": layer_name, "color": color}
+
+    def set_opacity(self, layer_name, opacity):
+        return {"name": "set_opacity", "layer_name": layer_name, "opacity": opacity}
+
+    def set_width(self, layer_name, width):
+        return {"name": "set_width", "layer_name": layer_name, "width": width}
+
+    def set_visibility(self, layer_name, visibility):
+        return {"name": "set_visibility", "layer_name": layer_name, "visibility": visibility}
+    
+    def __init__(self, model_version="gpt-3.5-turbo-0613"):
+        self.model_version = model_version
+        
+        self.function_descriptions = style_function_descriptions
+
+        self.messages = [
+            {
+                "role": "system",
+                "content": """You are a helpful assistant in styling a maplibre map. 
+                When asked to do something, you will call the appropriate function to style the map. Example: text mentioning 'color' should call
+                a function with 'color' in the name, or text containing the word 'opacity' or 'transparency' will refer to a function with 'opacity' in the name.
+                If you can't find the appropriate function, you will notify the user and ask them to provide 
+                text to instruct you to style the map.""",
+            },
+        ]
+        self.available_functions = {
+            "set_color": self.set_color,
+            "set_opacity": self.set_opacity,
+            "set_width": self.set_width,
+            "set_visibility": self.set_visibility,
+        }
+
+    def listen(self, message):
+        logging.info(f"In StyleAgent.listen()...message is: {message}")
+        
+        """Listen to a message from the user."""
+        #remove the last item in self.messages
+        if len(self.messages) > 1:
+            self.messages.pop()
+        self.messages.append({
+            "role": "user",
+            "content": message,
+        })
+        
+        # this will be the function gpt will call if it 
+        # determines that the user wants to call a function
+        function_response = None
+
+        try:
+            response = openai.ChatCompletion.create(
+                model=self.model_version,
+                messages=self.messages,
+                functions=self.function_descriptions,
+                function_call="auto",
+                temperature=0.1,
+                max_tokens=256,
+                top_p=1,
+                frequency_penalty=0,
+                presence_penalty=0
+            )
+
+            response_message = response["choices"][0]["message"]
+
+            logging.info(f"Response from OpenAI in StyleAgent: {response_message}")
+
+            if response_message.get("function_call"):
+                function_name = response_message["function_call"]["name"]
+                logging.info(f"Function name: {function_name}")
+                function_to_call = self.available_functions[function_name]
+                logging.info(f"Function to call: {function_to_call}")
+                function_args = json.loads(response_message["function_call"]["arguments"])
+                logging.info(f"Function args: {function_args}")
+                # determine the function to call
+                function_response = function_to_call(**function_args)
+                logging.info(f"Function response: {function_response}")
+
+                return {"response": function_response}
+            elif response_message.get("content"):
+                return {"response": response_message["content"]}
+            else:
+                return {"response": "I'm sorry, I don't understand."}
+        
+        except Exception as e:
+            return {"error": "Failed to get response from OpenAI in StyleAgent: " + str(e)}, 500
+        return {"response": function_response}

flask_app/app.py

@@ -0,0 +1,83 @@
+from flask import Flask, render_template, request, jsonify
+from flask_cors import CORS
+import os
+import openai
+from dotenv import load_dotenv
+import json
+import psycopg2
+from psycopg2 import sql
+from agents.marshall_agent import MarshallAgent
+from agents.navigation_agent import NavigationAgent
+from agents.style_agent import StyleAgent
+from agents.map_info_agent import MapInfoAgent
+import logging
+
+logging.basicConfig(level=logging.DEBUG,
+                    format='%(asctime)s %(levelname)s %(name)s %(threadName)s : %(message)s',
+                    handlers=[logging.StreamHandler()])
+
+load_dotenv()
+app = Flask(__name__)
+CORS(app)
+
+#openai.organization = os.getenv("OPENAI_ORGANIZATION")
+openai.api_key = os.getenv("OPENAI_API_KEY")
+model_version = os.getenv("OPENAI_MODEL_VERSION")
+UPLOAD_FOLDER = 'uploads/audio'
+
+navigation_agent = NavigationAgent(model_version=model_version)
+marshall_agent = MarshallAgent(model_version=model_version)
+style_agent = StyleAgent(model_version=model_version)
+map_info_agent = MapInfoAgent(model_version=model_version)
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+            
+@app.route('/ask', methods=['POST'])
+def ask():
+    message = request.json.get('message', '')
+    logging.info(f"Received message: {message}") 
+    return jsonify(marshall_agent.listen(message))
+
+@app.route('/navigate', methods=['POST'])
+def navigate():
+    message = request.json.get('message', '')
+    logging.info(f"Received message: {message}") 
+    return jsonify(navigation_agent.listen(message))
+
+@app.route('/layer', methods=['POST'])
+def layer():
+    message = request.json.get('message', '')
+    layer_names = request.json.get('layer_names', '')
+    logging.debug(f"In layer endpoint, layer_names are {layer_names} and message is {message}")  
+    return jsonify(map_info_agent.listen(message, layer_names))
+
+@app.route('/style', methods=['POST'])
+def style():
+    message = request.json.get('message', '')
+    layer_name = request.json.get('layer_name', '')
+    logging.debug(f"In style endpoint, layer_name is {layer_name} and message is {message}")  
+    prefixed_message = f"The following is referring to the layer {layer_name}."
+    prepended_message = prefixed_message + " " + message
+    logging.debug(f"In Style route, prepended_message is {prepended_message}")  
+    return jsonify(style_agent.listen(prepended_message))
+
+@app.route('/audio', methods=['POST'])
+def upload_audio():
+    audio_file = request.files['audio']
+    audio_file.save(os.path.join(UPLOAD_FOLDER, "user_audio.webm"))
+    audio_file=open(os.path.join(UPLOAD_FOLDER, "user_audio.webm"), 'rb')
+    transcript = openai.Audio.transcribe("whisper-1", audio_file)
+    logging.info(f"Received transcript: {transcript}")  
+    message = transcript['text']
+    #delete the audio
+    os.remove(os.path.join(UPLOAD_FOLDER, "user_audio.webm"))
+    return message
+
+@app.route('/select', methods=['POST'])
+def select():
+    return None
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0', port=5000)

flask_app/requirements.txt

@@ -0,0 +1,6 @@
+Flask
+requests==2.26.0
+openai
+python-dotenv
+flask_cors
+psycopg2-binary

flask_app/static/css/dark-theme.css

@@ -0,0 +1,70 @@
+.dark-body {
+    background-color: #121212;
+    color: #ffffff;
+}
+
+.dark-nav {
+    background-color: #333;
+    color: #FFF;
+}
+
+.dark-nav .navbar-brand {
+    font-size: 24px;  /* Adjust the size as you prefer */
+    color: #FFFFFF !important;  /* Important flag can ensure this color takes precedence */
+}
+
+
+.dark-container {
+    background-color: #121212;
+}
+
+.dark-map {
+    background-color: #333;
+}
+
+.dark-sidebar {
+    background-color: #222;
+}
+
+.dark-textarea {
+    background-color: #333;
+    color: #696767;
+    padding: 10px;
+}
+
+.dark-dropdown {
+    width: 100%;  /* This will make the dropdown take up the full width */
+    background-color: #333;
+    color: #FFF;
+    padding: 10px;
+}
+
+.dark-dropdown, .dark-input {
+    background-color: #333;
+    color: #FFF;
+    padding: 10px;
+}
+
+.dark-btn {
+    background-color: #444;
+    color: #FFF;
+    padding: 10px 20px;
+}
+
+.dark-btn:hover {
+    background-color: #555;
+}
+
+.dark-label {
+    color: #FFF;  /* White text */
+    font-size: 16px;  /* Adjust size as needed */
+    margin-bottom: 5px;  /* Spacing from the dropdown */
+}
+
+
+.example-commands {
+    color: #FFF;  /* White text */
+    font-size: 16px;  /* Adjust size as needed */
+    margin-top: 10px;  /* Spacing from the buttons */
+}
+

flask_app/static/css/main.css

@@ -0,0 +1,5 @@
+html, body, .container-fluid, .row, #map {
+    height: 100%;
+    margin: 0;
+    padding: 0;
+}

flask_app/static/js/map_info_response_agent.js

@@ -0,0 +1,11 @@
+class MapInfoResponseAgent extends ResponseAgent {
+    handleResponse(userMessage, response_data) {
+        console.log("In MapInfoResponseAgent.handleResponse...");
+        console.log("response_data: " + JSON.stringify(response_data));
+        if (response_data.name === "select_layer_name") {
+            const dropdown = document.getElementById('layerDropdown');
+            dropdown.value = response_data.layer_name;
+        } 
+        return this.getResponseString(userMessage, response_data);
+    }
+}

flask_app/static/js/navigation_response_agent.js

@@ -0,0 +1,31 @@
+class NavigationResponseAgent extends ResponseAgent {
+    handleResponse(userMessage, response_data) {
+        console.log("In NavigationResponseAgent.handleResponse...")
+        if (response_data.name === "go_to_location") {
+            this.map.flyTo({
+                center: [response_data.longitude, response_data.latitude],
+                zoom: 12
+            });
+        } else if (response_data.name === "zoom_in") {
+            this.map.zoomTo(this.map.getZoom() + response_data.zoom_levels);
+        } else if (response_data.name === "zoom_out") {
+            this.map.zoomTo(this.map.getZoom() - response_data.zoom_levels);
+        } else if (response_data.name === "pan_in_direction") {
+            const lnglat = this.map.getCenter();
+            var distance = 1;
+            if (response_data.distance_in_kilometers) {
+                distance = response_data.distance_in_kilometers;
+            }
+            const bearing = getBearing(response_data.direction);
+            if (bearing === null) {
+                return response_message;
+            }
+            const new_point = turf.destination(turf.point([lnglat.lng, lnglat.lat]), distance, bearing);
+            console.log(new_point);
+            this.map.panTo(new_point.geometry.coordinates, {
+                duration: 1000
+            });
+        }
+        return this.getResponseString(userMessage, response_data);
+    }
+}

flask_app/static/js/recorder.js

@@ -0,0 +1,62 @@
+class Recorder {
+    constructor(submitMessage, map) {
+        this.mediaRecorder = null;
+        this.audioChunks = [];
+        this.options = { mimeType: 'audio/webm' };
+        this.submitMessage = submitMessage;
+        this.map = map;
+        this.init();
+    }
+
+    async init() {
+        try {
+            const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+            if (!MediaRecorder.isTypeSupported(this.options.mimeType)) {
+                console.error(`${this.options.mimeType} is not supported`);
+                this.options.mimeType = ''; // Fallback to the default mimeType
+            }
+            this.mediaRecorder = new MediaRecorder(stream, this.options);
+            this.setupEventListeners();
+        } catch (err) {
+            console.error("Could not get media stream:", err);
+        }
+    }
+
+    setupEventListeners() {
+        this.mediaRecorder.ondataavailable = event => {
+            this.audioChunks.push(event.data);
+        };
+
+        this.mediaRecorder.onstop = async () => {
+            try {
+                const audioBlob = new Blob(this.audioChunks, { type: this.options.mimeType }); 
+                const formData = new FormData();
+                formData.append("audio", audioBlob);
+                const response = await fetch('/audio', {
+                    method: 'POST',
+                    body: formData
+                });
+                const userMessage = await response.text();
+                console.log("Audio uploaded:", userMessage);
+                this.submitMessage(this.map, userMessage); 
+                
+            } catch (error) {
+                console.error("Error uploading audio:", error);
+            }
+        };
+
+        document.getElementById("startRecord").addEventListener("click", () => {
+            if (this.mediaRecorder) {
+                this.audioChunks = [];
+                this.mediaRecorder.start();
+            }
+        });
+
+        document.getElementById("stopRecord").addEventListener("click", () => {
+            if (this.mediaRecorder) {
+                this.mediaRecorder.stop();
+            }
+        });
+    }
+}
+

flask_app/static/js/response_agent.js

@@ -0,0 +1,11 @@
+class ResponseAgent {
+    constructor(map) {
+        this.map = map;
+    }
+
+    getResponseString(userMessage, response_data) {
+        console.log("In ResponseAgent.getResponseString...")
+        console.log(response_data)
+        return "You: " + userMessage + "\nAI: " + JSON.stringify(response_data) + "\n" 
+    }
+}

flask_app/static/js/style_response_agent.js

@@ -0,0 +1,37 @@
+class StyleResponseAgent extends ResponseAgent {
+    handleResponse(userMessage, response_data) {
+        console.log("In StyleResponseAgent.handleResponse...")
+        const layer_type = this.map.getLayer(response_data.layer_name).type;
+        switch (response_data.name) {
+            case "set_color":
+                if (layer_type === "fill") {
+                    this.map.setPaintProperty(response_data.layer_name, 'fill-color', response_data.color);
+                } else if (layer_type === "line") {
+                    this.map.setPaintProperty(response_data.layer_name, 'line-color', response_data.color);
+                }
+                break;
+            case "set_width":
+                if (layer_type === "line") {
+                    this.map.setPaintProperty(response_data.layer_name, 'line-width', response_data.width);
+                }
+                break;
+            case "set_opacity":
+                if (layer_type === "line") {
+                    this.map.setPaintProperty(response_data.layer_name, 'line-opacity', response_data.opacity);
+                } else if (layer_type === "fill") {
+                    this.map.setPaintProperty(response_data.layer_name, 'fill-opacity', response_data.opacity);
+                }
+                break;
+            case "set_visibility":
+                if (response_data.visibility === "visible") {
+                    this.map.setLayoutProperty(response_data.layer_name, 'visibility', 'visible');
+                } else if (response_data.visibility === "none") {
+                    this.map.setLayoutProperty(response_data.layer_name, 'visibility', 'none');
+                }
+                break;
+            default:
+                break;
+        }
+        return this.getResponseString(userMessage, response_data);
+    }
+}

flask_app/static/js/utils.js

@@ -0,0 +1,43 @@
+function get_layer_names(map) {
+    var layer_names = [];
+    var layers = map.getStyle().layers;
+    for (var i = 0; i < layers.length; i++) {
+        layer_names.push(layers[i].id);
+    }
+    return layer_names;
+} 
+
+function getBearing(direction) {
+    if (direction == 'north') {
+        return 0;
+    } 
+    else if (direction == 'east') {
+        return 90;
+    } 
+    else if (direction == 'south') {
+        return 180;
+    }
+    else if (direction == 'west') {
+        return -90;
+    }
+    else if (direction == 'northeast') {
+        return 45;
+    }
+    else if (direction == 'southeast') {
+        return 135;
+    }
+    else if (direction == 'southwest') {
+        return -135;
+    } else if (direction == 'northwest') {
+        return -45;
+    } else {
+        return null;
+    }
+}
+
+function getResponseString(userMessage, response_data) {
+    console.log("In getResponseString...")
+    console.log(typeof response_data)
+    console.log(response_data)
+    return "You: " + userMessage + "\nAI: " + JSON.stringify(response_data) + "\n" 
+}

flask_app/static/sample-geojson.json

@@ -0,0 +1,34 @@
+{ "type": "FeatureCollection",
+  "features": [
+    { "type": "Feature",
+      "geometry": {"type": "Point", "coordinates": [102.0, 0.5]},
+      "properties": {"prop0": "value0"}
+      },
+    { "type": "Feature",
+      "geometry": {
+        "type": "LineString",
+        "coordinates": [
+          [102.0, 0.0], [103.0, 1.0], [104.0, 0.0], [105.0, 1.0]
+          ]
+        },
+      "properties": {
+        "prop0": "value0",
+        "prop1": 0.0
+        }
+      },
+    { "type": "Feature",
+       "geometry": {
+         "type": "Polygon",
+         "coordinates": [
+           [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0],
+             [100.0, 1.0], [100.0, 0.0] ]
+           ]
+
+       },
+       "properties": {
+         "prop0": "value0",
+         "prop1": {"this": "that"}
+         }
+       }
+    ]
+  }

flask_app/templates/index.html

@@ -0,0 +1,271 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css" rel="stylesheet">
+    <link href="https://unpkg.com/maplibre-gl@3.3.1/dist/maplibre-gl.css" rel="stylesheet">
+    <!--turfjs-->
+    <script src='https://unpkg.com/@turf/turf@6/turf.min.js'></script>
+    <link href="{{ url_for('static', filename='css/main.css') }}" rel="stylesheet">
+    <link href="{{ url_for('static', filename='css/dark-theme.css') }}" rel="stylesheet">
+    <title>MapGPT</title>
+</head>
+<body class="dark-body">
+
+<!-- Navbar -->
+<nav class="navbar navbar-expand-lg dark-nav">
+    <a class="navbar-brand" href="#">MapGPT</a>
+</nav>
+
+<!-- Map and Sidebar -->
+<div class="container-fluid dark-container">
+    <div class="row">
+        <!-- Map (assuming maplibre fills its container) -->
+        <div id="map" class="col-8 dark-map"></div>
+
+        <!-- Chat Sidebar -->
+        <div class="col-4 dark-sidebar">
+            <textarea id="chatbox" class="form-control dark-textarea my-3" rows="5" readonly></textarea>
+            <label for="layerDropdown" class="dark-label mx-2">Layers</label>
+            <select id="layerDropdown" class="dark-dropdown my-2">
+                <!-- Layers will be populated here -->
+            </select>
+            <input id="message" class="form-control dark-input my-2" placeholder="Type a message...">
+            <button id="send" class="btn dark-btn my-2">Send</button>
+            <!--button to clear the chatbox-->
+            <button id="clear" class="btn dark-btn my-2">Clear</button>
+            <!--audio recorder buttons-->
+            <div class="audio-recorder">
+                <button id="startRecord" class="btn dark-btn my-2">Start Recording</button>
+                <button id="stopRecord" class="btn dark-btn my-2">Stop Recording</button>
+            </div>
+            <!-- Example Commands -->
+            <div class="example-commands">
+                <p>Type or speak natural language commands to modify the stylesheet and navigate the map.</p>
+                <p>To modify the stylesheet, <b><i>select a layer on the map</i></b> or choose from the <b><i>Layers dropdown</i></b> before entering commands. 
+                    You can also type or speak something like 'select the buildings layer'. If a particular command seems to confuse the AI, try alternate phrasing.</p>
+                <p>Example commands:</p>
+                <ul>
+                    <li>Change color to green</li>
+                    <li>Turn layer off</li>
+                    <li>Go to Paris</li>
+                    <li>Where is the Statue of Liberty?</li>
+                    <li>Pan northeast 20km</li>
+                    <li>Change opacity to 50%</li>
+                    <li>line width 5.</li>
+            </div>
+        </div>
+    </div>
+</div>
+
+<!-- Scripts -->
+<script src="https://code.jquery.com/jquery-3.5.1.min.js"></script>
+<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
+<script src="https://unpkg.com/maplibre-gl@3.3.1/dist/maplibre-gl.js"></script>
+<script src="{{ url_for('static', filename='js/utils.js') }}"></script>
+<script src="{{ url_for('static', filename='js/response_agent.js') }}"></script>
+<script src="{{ url_for('static', filename='js/navigation_response_agent.js') }}"></script>
+<script src="{{ url_for('static', filename='js/style_response_agent.js') }}"></script>
+<script src="{{ url_for('static', filename='js/map_info_response_agent.js') }}"></script>
+<script src="{{ url_for('static', filename='js/recorder.js') }}"></script>
+<script>
+
+    document.addEventListener("DOMContentLoaded", function() {
+
+        // Turn the Start Recording button color to red when recording
+        const startRecordButton = document.getElementById('startRecord');
+        const stopRecordButton = document.getElementById('stopRecord');
+        if (startRecordButton) {
+            startRecordButton.addEventListener('click', function() {
+                //make the background color a maroonish color
+                startRecordButton.style.backgroundColor = '#800000';
+            });
+        }
+        if (stopRecordButton) {
+            stopRecordButton.addEventListener('click', function() {
+                startRecordButton.style.backgroundColor = '#444';
+            });
+        }
+
+
+        var map = new maplibregl.Map({
+            container: 'map',
+            style: "https://api.maptiler.com/maps/basic-v2/style.json?key=oDgyOnjlr9LH9XmQJNBF",
+            //style: "https://tileserver-rbt-agc-dev.apps.kubic.dev.ngaxc.net/styles/RBT-JOG-3395/style.json",
+            center: [24.6032, 56.8796],
+            zoom: 11
+        });
+
+        map.on('click', (e) => {
+            try {
+                //get the id of the layer that was clicked
+                var layerId = map.queryRenderedFeatures(e.point)[0].layer.id;
+                //set the value of the dropdown to the layer id
+                document.getElementById('layerDropdown').value = layerId;
+            } catch (error) {
+                //if no layer was clicked, then return
+                return;
+            }
+        });
+
+        // Add navigation control (zoom buttons)
+        map.addControl(new maplibregl.NavigationControl());
+
+        // Add our response agents
+        const navigationResponseAgent = new NavigationResponseAgent(map);
+        const styleResponseAgent = new StyleResponseAgent(map);
+        const mapInfoResponseAgent = new MapInfoResponseAgent(map);
+
+        const sendButton = document.getElementById('send');
+        const messageBox = document.getElementById('message');
+        const chatbox = document.getElementById('chatbox');
+        const clearButton = document.getElementById('clear');
+        
+        sendButton.addEventListener('click', function() {
+            submitMessage(map, messageBox.value);
+        });
+
+        clearButton.addEventListener('click', function() {
+            chatbox.value = "";
+        });
+
+        map.on('load', function () {
+            populateLayersDropdown();
+        });
+
+        document.getElementById("message").addEventListener("keyup", function(event) {
+            if (event.keyCode === 13) { // Enter key code
+                event.preventDefault(); // Prevent the default form submission
+                submitMessage(map, messageBox.value);
+            }
+        });
+
+        function populateLayersDropdown() {
+            const dropdown = document.getElementById('layerDropdown');
+            const layers = map.getStyle().layers;
+
+            layers.forEach(layer => {
+                const option = document.createElement('option');
+                option.value = layer.id;
+                option.innerText = layer.id;
+                dropdown.appendChild(option);
+            });
+        }
+
+        //function to submit a message to the chatbox
+        function submitMessage(map, userMessage) {
+
+            const chatbox = document.getElementById('chatbox');
+            const messageBox = document.getElementById('message');
+
+            if (userMessage.trim() === "") {
+                // Prevent sending empty messages
+                return;
+            }
+
+            fetch('/ask', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify({message: userMessage})
+            })
+            .then(response => response.json())
+            .then(data => {
+                response_data = data.response;
+                console.log(data);
+                handle_choose_agent_response(userMessage, response_data);
+
+                return;
+
+            })
+            .catch(error => {
+                console.error("Error:", error);
+                chatbox.value += "Error sending message.\n";
+            });
+            messageBox.value = "";
+        }
+
+        function handle_choose_agent_response(userMessage, response_data) {
+            const dropdown = document.getElementById('layerDropdown');
+            const chatbox = document.getElementById('chatbox');
+            console.log(`Response data in handle_choose_agent_response: ${JSON.stringify(response_data)}`);
+            if (response_data.agent_name === 'NavigationAgent') {
+                fetch('/navigate', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({message: userMessage})
+                })
+                .then(response => response.json())
+                .then(data => {
+                    response_data = data.response;
+                    console.log(data);
+                    chatbox.value = navigationResponseAgent.handleResponse(userMessage, response_data);
+                    return;
+
+                })
+                .catch(error => {
+                    console.error("Error:", error);
+                    chatbox.value += "Error sending message.\n";
+                });
+            } else if (response_data.agent_name === 'StyleAgent') {
+                const layer_name = dropdown.options[dropdown.selectedIndex].text;
+                const bodytext = JSON.stringify({message: userMessage, layer_name: layer_name});
+                console.log(`Bodytext: ${bodytext}`);
+                fetch('/style', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({message: userMessage, layer_name: layer_name})
+                })
+                .then(response => response.json())
+                .then(data => {
+                    response_data = data.response;
+                    console.log(data);
+                    chatbox.value = styleResponseAgent.handleResponse(userMessage, response_data);
+                    return;
+
+                })
+                .catch(error => {
+                    console.error("Error:", error);
+                    chatbox.value += "Error sending message.\n";
+                });
+            } else if (response_data.agent_name === 'MapInfoAgent') {
+                const layer_names = get_layer_names(map);
+                fetch('/layer', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({message: userMessage, layer_names: layer_names})
+                })
+                .then(response => response.json())
+                .then(data => {
+                    response_data = data.response;
+                    console.log(response_data);
+                    chatbox.value = mapInfoResponseAgent.handleResponse(userMessage, response_data);
+                    return;
+                })
+                .catch(error => {
+                    console.error("Error:", error);
+                    chatbox.value += "Error sending message.\n";
+                });
+            } else {
+                console.log("Unknown agent");
+                chatbox.value = "I'm sorry, can you rephrase that?";
+            }
+
+            return getResponseString(userMessage, response_data);
+        }
+
+        // Add our recorder
+        const recorder = new Recorder(submitMessage, map);
+    });
+</script>
+
+</body>
+</html>

flask_app/utils/database.py

@@ -0,0 +1,35 @@
+class Database:
+    def __init__(self, conn):
+        self.conn = conn
+
+    def get_table_names(self):
+        table_names = []
+        query = "SELECT name FROM sqlite_schema WHERE type='table';"
+        tables = self.execute(query)
+        for table in tables:
+            table_names.append(table[0])
+        return table_names
+
+    def get_column_names(self, table_name):
+        column_names = []
+        query = f"PRAGMA table_info({table_name});"
+        columns = self.execute(query)
+        for column in columns:
+            column_names.append(column[1])
+        return column_names
+
+    def get_database_info(self):
+        database_info = []
+        for table_name in self.get_table_names():
+            column_names = self.get_column_names(table_name)
+            database_info.append(
+                {"table_name": table_name, "column_names": column_names}
+            )
+        return database_info
+
+    def execute(self, query):
+        res = self.conn.execute(query)
+        return res.fetchall()
+
+    def close(self):
+        self.conn.close()