mirror
/
planetiler
kopia lustrzana https://github.com/onthegomap/planetiler
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Page through S3 index (#608)

pull/609/head
Michael Barry 2023-06-21 20:43:37 -04:00 zatwierdzone przez GitHub
rodzic 6203387b59
commit a48a9c8391
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
2 zmienionych plików z 216 dodań i 208 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

49
planetiler-core/src/main/java/com/onthegomap/planetiler/util/AwsOsm.java
Wyświetl plik

@ -9,8 +9,12 @@ import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import javax.annotation.concurrent.Immutable;
/**
@ -19,12 +23,13 @@ import javax.annotation.concurrent.Immutable;
* <a href="https://overturemaps.org">Overture Maps Foundation</a>.
*/
public class AwsOsm {
private static final int MAX_PAGES = 100;
public static final AwsOsm OSM_PDS = new AwsOsm("https://osm-pds.s3.amazonaws.com/");
public static final AwsOsm OVERTURE = new AwsOsm("https://overturemaps-us-west-2.s3.amazonaws.com/");
private static final ObjectMapper mapper = new XmlMapper().registerModule(new Jdk8Module());
private final String bucketIndexUrl;
private volatile IndexXml index = null;
private final CopyOnWriteArrayList<ContentXml> entries = new CopyOnWriteArrayList<>();
protected AwsOsm(String bucketIndexUrl) {
this.bucketIndexUrl = bucketIndexUrl;
@ -42,34 +47,46 @@ public class AwsOsm {
* @throws IllegalArgumentException if no matches, or more than one match is found.
*/
public String getDownloadUrl(String searchQuery, PlanetilerConfig config) {
IndexXml indexXml = getAndCacheIndex(config);
var indexXml = getAndCacheIndex(config);
return searchIndexForDownloadUrl(searchQuery, indexXml);
}
private synchronized IndexXml getAndCacheIndex(PlanetilerConfig config) {
if (index == null) {
try (InputStream inputStream = Downloader.openStream(bucketIndexUrl, config)) {
index = parseIndexXml(inputStream);
} catch (IOException e) {
throw new IllegalStateException(e);
}
private synchronized List<ContentXml> getAndCacheIndex(PlanetilerConfig config) {
if (entries.isEmpty()) {
List<ContentXml> result = new ArrayList<>();
String nextPageParam = "";
int pageNum = 0;
do {
try (InputStream inputStream = Downloader.openStream(bucketIndexUrl + "?list-type=2" + nextPageParam, config)) {
if (pageNum++ > MAX_PAGES) {
throw new IllegalArgumentException("Too many entries in " + bucketIndexUrl + " to page through");
}
var page = parseIndexXml(inputStream);
result.addAll(page.contents());
nextPageParam = (!page.truncated() || page.nextToken() == null) ? null :
"&continuation-token=" + URLEncoder.encode(page.nextToken(), StandardCharsets.UTF_8);
} catch (IOException e) {
throw new IllegalStateException(e);
}
} while (nextPageParam != null);
entries.addAll(result);
}
return index;
return entries;
}
protected IndexXml parseIndexXml(InputStream indexXmlContent) throws IOException {
return mapper.readValue(indexXmlContent, IndexXml.class);
}
protected String searchIndexForDownloadUrl(String searchQuery, IndexXml index) {
protected String searchIndexForDownloadUrl(String searchQuery, List<ContentXml> index) {
if ("latest".equalsIgnoreCase(searchQuery)) {
return index.contents.stream()
return index.stream()
.filter(c -> c.key.endsWith(".osm.pbf"))
.map(c -> bucketIndexUrl + c.key)
.max(Comparator.naturalOrder())
.orElseThrow(() -> new IllegalArgumentException("Unable to find latest AWS osm download URL"));
} else {
List<String> results = index.contents.stream()
List<String> results = index.stream()
.filter(c -> c.key.endsWith("/planet-" + searchQuery + ".osm.pbf"))
.map(c -> bucketIndexUrl + c.key)
.toList();
@ -86,7 +103,11 @@ public class AwsOsm {
@Immutable
record IndexXml(
@JacksonXmlProperty(localName = "Contents")
@JacksonXmlElementWrapper(useWrapping = false) List<ContentXml> contents
@JacksonXmlElementWrapper(useWrapping = false) List<ContentXml> contents,
@JacksonXmlProperty(localName = "NextContinuationToken")
@JacksonXmlElementWrapper(useWrapping = false) String nextToken,
@JacksonXmlProperty(localName = "IsTruncated")
@JacksonXmlElementWrapper(useWrapping = false) boolean truncated
) {}
@JsonIgnoreProperties(ignoreUnknown = true)

375
planetiler-core/src/test/java/com/onthegomap/planetiler/util/AwsOsmTest.java
Wyświetl plik

@ -2,6 +2,7 @@ package com.onthegomap.planetiler.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -14,150 +15,162 @@ class AwsOsmTest {
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>osm-pds</Name>
<Prefix/>
<Marker/>
<NextContinuationToken>nextpage</NextContinuationToken>
<KeyCount>1000</KeyCount>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<IsTruncated>true</IsTruncated>
<Contents>
<Key>2021/planet-210830.orc</Key>
<LastModified>2021-09-05T00:53:27.000Z</LastModified>
<ETag>"3e38c5a4c1db83a70abaafb3e6784d51-818"</ETag>
<Size>85742946714</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<Key>2023/planet-230102.orc</Key>
<LastModified>2023-01-08T11:34:39.000Z</LastModified>
<ETag>"49a5e81ff968d16a0ac6e0b78c4e3303-924"</ETag>
<Size>96802097226</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2021/planet-210830.osm.pbf</Key>
<LastModified>2021-09-04T13:02:46.000Z</LastModified>
<ETag>"4373b6cd606f6c6f7d357e3b3c77ba6f-7632"</ETag>
<Size>64018213652</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<Key>2023/planet-230102.osm.pbf</Key>
<LastModified>2023-01-07T23:03:45.000Z</LastModified>
<ETag>"9664609944bd7022aa86c6ec390f0fe0-8568"</ETag>
<Size>71872573268</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2021/planet-210830.osm.pbf.md5</Key>
<LastModified>2021-09-04T13:00:10.000Z</LastModified>
<ETag>"6555b62666e14a24323d13b34126e5b3"</ETag>
<Key>2023/planet-230102.osm.pbf.md5</Key>
<LastModified>2023-01-07T23:00:11.000Z</LastModified>
<ETag>"a9681d16f0011d7126b99327ac6d1471"</ETag>
<Size>56</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2021/planet-210830.osm.pbf.torrent</Key>
<LastModified>2021-09-04T13:02:46.000Z</LastModified>
<ETag>"b2b82d568dec00f5e361358530672f25"</ETag>
<Size>306696</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<Key>2023/planet-230102.osm.pbf.torrent</Key>
<LastModified>2023-01-07T23:03:45.000Z</LastModified>
<ETag>"74513fffa6e06ff7acac9cd1f59d514b"</ETag>
<Size>344019</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2021/planet-210830.osm.pbf.torrent.md5</Key>
<LastModified>2021-09-04T13:00:10.000Z</LastModified>
<ETag>"5e0eaf44e8a69e7cac706d0a052a5356"</ETag>
<Key>2023/planet-230102.osm.pbf.torrent.md5</Key>
<LastModified>2023-01-07T23:00:11.000Z</LastModified>
<ETag>"6943a7a5dcddd146e3906e71e343e489"</ETag>
<Size>277</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2021/planet-210906.orc</Key>
<LastModified>2021-09-12T00:35:53.000Z</LastModified>
<ETag>"a89c489c799019c5e5f914ef5ba5c030-820"</ETag>
<Size>85890640680</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<Key>2023/planet-230109.orc</Key>
<LastModified>2023-01-15T13:01:34.000Z</LastModified>
<ETag>"42a76f8c06ec62f629afc34bbf3b09bb-925"</ETag>
<Size>96942728676</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2021/planet-210906.osm.pbf</Key>
<LastModified>2021-09-11T13:47:44.000Z</LastModified>
<ETag>"c0c1a0ffdf1dd6ece9915bc7a568dfd3-7645"</ETag>
<Size>64122989442</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<Key>2023/planet-230109.osm.pbf</Key>
<LastModified>2023-01-15T03:17:15.000Z</LastModified>
<ETag>"19d28987233519932b1c1e25c8c5663f-8580"</ETag>
<Size>71970123667</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2021/planet-210906.osm.pbf.md5</Key>
<LastModified>2021-09-11T13:45:10.000Z</LastModified>
<ETag>"738b54c550a99704b47b18e7afe287dc"</ETag>
<Key>2023/planet-230109.osm.pbf.md5</Key>
<LastModified>2023-01-15T03:15:11.000Z</LastModified>
<ETag>"c53f91a851336234467e1c11750bc1dc"</ETag>
<Size>56</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2021/planet-210906.osm.pbf.torrent</Key>
<LastModified>2021-09-11T13:47:44.000Z</LastModified>
<ETag>"c734185715f555666acff7fa3d1dd504"</ETag>
<Size>307196</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<Key>2023/planet-230109.osm.pbf.torrent</Key>
<LastModified>2023-01-15T03:17:15.000Z</LastModified>
<ETag>"145fb95bb3e2d2b0ba38280ebddbe881"</ETag>
<Size>344499</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2021/planet-210906.osm.pbf.torrent.md5</Key>
<LastModified>2021-09-11T13:45:10.000Z</LastModified>
<ETag>"5e0eaf44e8a69e7cac706d0a052a5356"</ETag>
<Key>2023/planet-230109.osm.pbf.torrent.md5</Key>
<LastModified>2023-01-15T03:15:11.000Z</LastModified>
<ETag>"6943a7a5dcddd146e3906e71e343e489"</ETag>
<Size>277</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>changesets/changesets-latest.orc</Key>
<LastModified>2021-09-11T13:02:24.000Z</LastModified>
<ETag>"a63ce01a6dd033c94c862a690f10c9e3-468"</ETag>
<Size>3917489634</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<Key>2023/planet-230116.orc</Key>
<LastModified>2023-01-23T03:05:03.000Z</LastModified>
<ETag>"e2d2c430850d6804e2d5c6e3fccfd0fa-927"</ETag>
<Size>97157879124</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>planet-history/history-latest.orc</Key>
<LastModified>2021-09-12T21:17:59.000Z</LastModified>
<ETag>"322389f0eb7dd3fe3d597c9383fee4af-8620"</ETag>
<Size>144609225892</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<Key>2023/planet-230116.osm.pbf</Key>
<LastModified>2023-01-22T15:32:20.000Z</LastModified>
<ETag>"27af8ae8a3614a8a6a8300a3de93991f-8599"</ETag>
<Size>72127252191</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>planet/planet-latest.orc</Key>
<LastModified>2021-09-12T11:08:29.000Z</LastModified>
<ETag>"4ce8adc7208a0423cd4bb48ca087bf76-5120"</ETag>
<Size>85890640680</Size>
<Owner>
<ID>3555355acefe4c7705bd24eb6def06f2564b97fa9b5fe5bc435443a21476f67d</ID>
<DisplayName>seth+osmpds</DisplayName>
</Owner>
<Key>2023/planet-230116.osm.pbf.md5</Key>
<LastModified>2023-01-22T15:30:10.000Z</LastModified>
<ETag>"6943a7a5dcddd146e3906e71e343e489"</ETag>
<Size>277</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2023/planet-230116.osm.pbf.torrent</Key>
<LastModified>2023-01-22T16:15:23.000Z</LastModified>
<ETag>"95e1808f844c7a657e53ceae02166ece"</ETag>
<Size>345239</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2023/planet-230116.osm.pbf.torrent.md5</Key>
<LastModified>2023-01-22T16:15:10.000Z</LastModified>
<ETag>"6943a7a5dcddd146e3906e71e343e489"</ETag>
<Size>277</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2023/planet-230123.orc</Key>
<LastModified>2023-01-30T05:44:28.000Z</LastModified>
<ETag>"535fabb24c948525d8a807a7f9cc9fc3-929"</ETag>
<Size>97310366433</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2023/planet-230123.osm.pbf</Key>
<LastModified>2023-01-29T13:02:20.000Z</LastModified>
<ETag>"b871759f59e0abb925d0bf372a4acf89-8611"</ETag>
<Size>72232247230</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2023/planet-230123.osm.pbf.md5</Key>
<LastModified>2023-01-29T13:00:10.000Z</LastModified>
<ETag>"31e1303c9c35eb043cb3da55ef082de5"</ETag>
<Size>56</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2023/planet-230123.osm.pbf.torrent</Key>
<LastModified>2023-01-29T13:02:20.000Z</LastModified>
<ETag>"903aecd263c00856d7441bce9550a9d5"</ETag>
<Size>345739</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2023/planet-230123.osm.pbf.torrent.md5</Key>
<LastModified>2023-01-29T13:00:10.000Z</LastModified>
<ETag>"6943a7a5dcddd146e3906e71e343e489"</ETag>
<Size>277</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2023/planet-230130.orc</Key>
<LastModified>2023-02-06T14:07:57.000Z</LastModified>
<ETag>"dead001d3a54054e9c81d1312df62ff9-930"</ETag>
<Size>97476386631</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>2023/planet-230130.osm.pbf</Key>
<LastModified>2023-02-05T12:47:17.000Z</LastModified>
<ETag>"5c227c0151b5d1ccae1713b0c7b63427-8625"</ETag>
<Size>72344054646</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
</ListBucketResult>
@ -165,90 +178,61 @@ class AwsOsmTest {
private static final byte[] overtureResponse =
"""
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>overturemaps-us-west-2</Name>
<Prefix />
<Marker />
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-cook_county_il.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:47:32.000Z</LastModified>
<ETag>"89e67e64e866c56db9abe700652bd253-17"</ETag>
<Size>138598884</Size>
<Owner>
<ID>15b7f69019aeb60070a7e5e0abef900903bcad93b2cd252fc9ac14881f1cea14</ID>
<DisplayName>sethfitz+overture-data-distribution</DisplayName>
</Owner>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-eastern_ma.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:47:57.000Z</LastModified>
<ETag>"983fbf7d59ea77478851059edf20ae82-17"</ETag>
<Size>139513514</Size>
<Owner>
<ID>15b7f69019aeb60070a7e5e0abef900903bcad93b2cd252fc9ac14881f1cea14</ID>
<DisplayName>sethfitz+overture-data-distribution</DisplayName>
</Owner>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-king_county_wa.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:48:49.000Z</LastModified>
<ETag>"fb9b875482136a1d77b8722aa0ccdd49-8"</ETag>
<Size>60249069</Size>
<Owner>
<ID>15b7f69019aeb60070a7e5e0abef900903bcad93b2cd252fc9ac14881f1cea14</ID>
<DisplayName>sethfitz+overture-data-distribution</DisplayName>
</Owner>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-maricopa_and_pinal_counties_az.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:49:12.000Z</LastModified>
<ETag>"fd0ac4f256e5e4937abdc471486637f6-15"</ETag>
<Size>125059646</Size>
<Owner>
<ID>15b7f69019aeb60070a7e5e0abef900903bcad93b2cd252fc9ac14881f1cea14</ID>
<DisplayName>sethfitz+overture-data-distribution</DisplayName>
</Owner>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-orange_county_fl.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:49:59.000Z</LastModified>
<ETag>"3e1aa7ac82267d947daf3541f91cdb6d-4"</ETag>
<Size>25542787</Size>
<Owner>
<ID>15b7f69019aeb60070a7e5e0abef900903bcad93b2cd252fc9ac14881f1cea14</ID>
<DisplayName>sethfitz+overture-data-distribution</DisplayName>
</Owner>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-santa_clara_county_ca.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:50:10.000Z</LastModified>
<ETag>"1a104df658f2667107a5d20ac4b503f2-8"</ETag>
<Size>64331924</Size>
<Owner>
<ID>15b7f69019aeb60070a7e5e0abef900903bcad93b2cd252fc9ac14881f1cea14</ID>
<DisplayName>sethfitz+overture-data-distribution</DisplayName>
</Owner>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/planet-2023-04-02-alpha.osm.pbf</Key>
<LastModified>2023-04-05T16:24:04.000Z</LastModified>
<ETag>"a45f7016445256b2735f7765c7668e87-5496"</ETag>
<Size>92196566177</Size>
<Owner>
<ID>15b7f69019aeb60070a7e5e0abef900903bcad93b2cd252fc9ac14881f1cea14</ID>
<DisplayName>sethfitz+overture-data-distribution</DisplayName>
</Owner>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Name>overturemaps-us-west-2</Name>
<Prefix/>
<KeyCount>7</KeyCount>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-cook_county_il.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:47:32.000Z</LastModified>
<ETag>"89e67e64e866c56db9abe700652bd253-17"</ETag>
<Size>138598884</Size>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-eastern_ma.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:47:57.000Z</LastModified>
<ETag>"983fbf7d59ea77478851059edf20ae82-17"</ETag>
<Size>139513514</Size>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-king_county_wa.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:48:49.000Z</LastModified>
<ETag>"fb9b875482136a1d77b8722aa0ccdd49-8"</ETag>
<Size>60249069</Size>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-maricopa_and_pinal_counties_az.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:49:12.000Z</LastModified>
<ETag>"fd0ac4f256e5e4937abdc471486637f6-15"</ETag>
<Size>125059646</Size>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-orange_county_fl.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:49:59.000Z</LastModified>
<ETag>"3e1aa7ac82267d947daf3541f91cdb6d-4"</ETag>
<Size>25542787</Size>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/building-extracts/2023-04-02-alpha-santa_clara_county_ca.geojsonseq.gz</Key>
<LastModified>2023-04-13T16:50:10.000Z</LastModified>
<ETag>"1a104df658f2667107a5d20ac4b503f2-8"</ETag>
<Size>64331924</Size>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
<Contents>
<Key>release/2023-04-02-alpha/planet-2023-04-02-alpha.osm.pbf</Key>
<LastModified>2023-04-05T16:24:04.000Z</LastModified>
<ETag>"a45f7016445256b2735f7765c7668e87-5496"</ETag>
<Size>92196566177</Size>
<StorageClass>INTELLIGENT_TIERING</StorageClass>
</Contents>
</ListBucketResult>
"""
.getBytes(StandardCharsets.UTF_8);
@ -257,26 +241,29 @@ class AwsOsmTest {
void testFound() throws IOException {
var awsOsm = new AwsOsm("https://base.url/");
var index = awsOsm.parseIndexXml(new ByteArrayInputStream(response));
assertEquals("https://base.url/2021/planet-210906.osm.pbf",
awsOsm.searchIndexForDownloadUrl("210906", index));
assertEquals("https://base.url/2021/planet-210830.osm.pbf",
awsOsm.searchIndexForDownloadUrl("210830", index));
assertEquals("nextpage", index.nextToken());
assertTrue(index.truncated());
assertEquals("https://base.url/2023/planet-230102.osm.pbf",
awsOsm.searchIndexForDownloadUrl("230102", index.contents()));
assertEquals("https://base.url/2023/planet-230116.osm.pbf",
awsOsm.searchIndexForDownloadUrl("230116", index.contents()));
}
@Test
void testLatest() throws IOException {
var awsOsm = new AwsOsm("https://base.url/");
var index = awsOsm.parseIndexXml(new ByteArrayInputStream(response));
String url = awsOsm.searchIndexForDownloadUrl("latest", index);
assertEquals("https://base.url/2021/planet-210906.osm.pbf", url);
String url = awsOsm.searchIndexForDownloadUrl("latest", index.contents());
assertEquals("https://base.url/2023/planet-230130.osm.pbf", url);
}
@Test
void testNotFound() throws IOException {
var awsOsm = new AwsOsm("https://base.url/");
var index = awsOsm.parseIndexXml(new ByteArrayInputStream(response));
var contents = index.contents();
assertThrows(IllegalArgumentException.class,
() -> awsOsm.searchIndexForDownloadUrl("1231", index));
() -> awsOsm.searchIndexForDownloadUrl("1231", contents));
}
@Test
@ -285,11 +272,11 @@ class AwsOsmTest {
var index = awsOsm.parseIndexXml(new ByteArrayInputStream(overtureResponse));
assertEquals(
"https://base.url/release/2023-04-02-alpha/planet-2023-04-02-alpha.osm.pbf",
awsOsm.searchIndexForDownloadUrl("latest", index)
awsOsm.searchIndexForDownloadUrl("latest", index.contents())
);
assertEquals(
"https://base.url/release/2023-04-02-alpha/planet-2023-04-02-alpha.osm.pbf",
awsOsm.searchIndexForDownloadUrl("2023-04-02-alpha", index)
awsOsm.searchIndexForDownloadUrl("2023-04-02-alpha", index.contents())
);
}
}