diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/actions/AddMapWithAILayerAction.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/actions/AddMapWithAILayerAction.java
index 80c60aa..21b0b83 100644
--- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/actions/AddMapWithAILayerAction.java
+++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/actions/AddMapWithAILayerAction.java
@@ -90,7 +90,7 @@ public class AddMapWithAILayerAction extends JosmAction implements AdaptableActi
                 }
             }));
         }
-
+        MapWithAIDataUtils.getLayer(false).addDownloadedInfo(info);
     }
 
     @Override
@@ -100,7 +100,8 @@ public class AddMapWithAILayerAction extends JosmAction implements AdaptableActi
 
     @Override
     protected void updateEnabledState() {
-        setEnabled(!info.isBlacklisted());
+        setEnabled(!info.isBlacklisted() && (MapWithAIDataUtils.getLayer(false) == null
+                || !MapWithAIDataUtils.getLayer(false).hasDownloaded(info)));
     }
 
     @Override
diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIDataUtils.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIDataUtils.java
index 3cb614d..74ba6a0 100644
--- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIDataUtils.java
+++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIDataUtils.java
@@ -199,7 +199,7 @@ public final class MapWithAIDataUtils {
         final List<Bounds> realBounds = realBBoxes.stream()
                 .flatMap(tBBox -> MapWithAIDataUtils.reduceBBoxSize(tBBox, maximumDimensions).stream())
                 .map(MapWithAIDataUtils::bboxToBounds).collect(Collectors.toList());
-        if (!MapWithAILayerInfo.getInstance().getLayers().isEmpty()) {
+        if (!MapWithAIPreferenceHelper.getMapWithAIUrl().isEmpty()) {
             if ((realBBoxes.size() < TOO_MANY_BBOXES) || confirmBigDownload(realBBoxes)) {
                 final PleaseWaitProgressMonitor monitor = new PleaseWaitProgressMonitor();
                 monitor.beginTask(tr("Downloading {0} Data", MapWithAIPlugin.NAME), realBounds.size());
diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAILayer.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAILayer.java
index 94e10c9..bc5494b 100644
--- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAILayer.java
+++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAILayer.java
@@ -9,7 +9,9 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -58,6 +60,7 @@ public class MapWithAILayer extends OsmDataLayer implements ActiveLayerChangeLis
     private Boolean switchLayers;
     private boolean continuousDownload = true;
     private final Lock lock;
+    private final HashSet<MapWithAIInfo> downloadedInfo = new HashSet<>();
 
     /**
      * Create a new MapWithAI layer
@@ -318,4 +321,32 @@ public class MapWithAILayer extends OsmDataLayer implements ActiveLayerChangeLis
         }
 
     }
+
+    /**
+     * Check if the layer has downloaded a specific data type
+     *
+     * @param info The info to check
+     * @return {@code true} if the info has been added to the layer
+     */
+    public boolean hasDownloaded(MapWithAIInfo info) {
+        return downloadedInfo.contains(info);
+    }
+
+    /**
+     * Indicate an info has been downloaded in this layer
+     *
+     * @param info The info that has been downloaded
+     */
+    public void addDownloadedInfo(MapWithAIInfo info) {
+        downloadedInfo.add(info);
+    }
+
+    /**
+     * Get the info that has been downloaded into this layer
+     *
+     * @return An unmodifiable collection of the downloaded info
+     */
+    public Collection<MapWithAIInfo> getDownloadedInfo() {
+        return Collections.unmodifiableCollection(downloadedInfo);
+    }
 }
diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIPreferenceHelper.java b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIPreferenceHelper.java
index 8cb61af..9bc00cb 100644
--- a/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIPreferenceHelper.java
+++ b/src/main/java/org/openstreetmap/josm/plugins/mapwithai/backend/MapWithAIPreferenceHelper.java
@@ -2,6 +2,7 @@
 package org.openstreetmap.josm.plugins.mapwithai.backend;
 
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -38,11 +39,16 @@ public final class MapWithAIPreferenceHelper {
      * @return A list of enabled MapWithAI urls (maps have source, parameters,
      *         enabled, and the url)
      */
-    public static List<MapWithAIInfo> getMapWithAIUrl() {
-        return MapWithAIDataUtils.getLayer(false) == null
-                || MapWithAIDataUtils.getLayer(false).getMapWithAIUrl() == null
-                        ? MapWithAILayerInfo.getInstance().getLayers()
-                        : Collections.singletonList(MapWithAIDataUtils.getLayer(false).getMapWithAIUrl());
+    public static Collection<MapWithAIInfo> getMapWithAIUrl() {
+        MapWithAILayer layer = MapWithAIDataUtils.getLayer(false);
+        if (layer != null) {
+            if (!layer.getDownloadedInfo().isEmpty()) {
+                return layer.getDownloadedInfo();
+            } else if (layer.getMapWithAIUrl() != null) {
+                return Collections.singleton(layer.getMapWithAIUrl());
+            }
+        }
+        return MapWithAILayerInfo.getInstance().getLayers();
     }
 
     /**