diff --git a/src/AddressData.Core/Constants.cs b/src/AddressData.Core/Constants.cs
index 009064e..84eef2f 100644
--- a/src/AddressData.Core/Constants.cs
+++ b/src/AddressData.Core/Constants.cs
@@ -7,16 +7,21 @@ public static class Constants
/// Retrieved from https://wiki.openstreetmap.org/wiki/Overpass_API#Public_Overpass_API_instances
///
public const string OverpassTurboUrl = "https://overpass-api.de/api/interpreter";
+
+ ///
+ /// User Agent is expected as per https://wiki.openstreetmap.org/wiki/Overpass_API#Public_Overpass_API_instances
+ ///
+ public const string OverpassTurboUserAgent = "AddressData/1.0";
public const string OverpassTurboGetAllCitiesQuery =
- "?data=[out:csv(::id,\"name\",\"name:en\";true;\",\")];area[place=\"city\"];out;";
+ "[out:csv(::id,\"name\",\"name:en\";true;\",\")];area[place=\"city\"];out;";
public static string OverpassTurboGetCityQuery(long areaId) =>
- $"?data=[out:csv(::id,\"name\",\"name:en\";true;\",\")];area({areaId});out;";
+ $"[out:csv(::id,\"name\",\"name:en\";true;\",\")];area({areaId});out;";
public static string OverpassTurboGetLatitudeLongitudeQuery(long areaId) =>
- $"?data=[out:csv(::lat, ::lon;true;\",\")];area({areaId})->.a;node(area.a);out 1;";
+ $"[out:csv(::lat, ::lon;true;\",\")];area({areaId})->.a;node(area.a);out 1;";
public static string OverpassTurboGetAddressesQuery(long areaId) =>
- $"?data=[out:csv(::lat, ::lon, \"addr:housenumber\", \"addr:street\", \"addr:postcode\";true;\",\")];area({areaId});nwr(area)[\"addr:housenumber\"][\"addr:street\"][\"addr:postcode\"];out center;";
+ $"[out:csv(::lat, ::lon, \"addr:housenumber\", \"addr:street\", \"addr:postcode\";true;\",\")];area({areaId});nwr(area)[\"addr:housenumber\"][\"addr:street\"][\"addr:postcode\"];out center;";
public static string OverpassTurboAreaInfoQuery(string latitude, string longitude) =>
- $"?data=[out:json];is_in({latitude},{longitude})->.a;area.a[name][boundary=administrative][admin_level=2];out tags;area.a[name][boundary=administrative][admin_level=4];out tags;";
+ $"[out:json];is_in({latitude},{longitude})->.a;area.a[name][boundary=administrative][admin_level=2];out tags;area.a[name][boundary=administrative][admin_level=4];out tags;";
// Overpass Turbo Response
public const string OverpassTurboResponseElements = "elements";
@@ -33,6 +38,6 @@ public static string OverpassTurboAreaInfoQuery(string latitude, string longitud
public const string ErrorControllerRoute = "error";
// Other settings
- public const int MinimumNumberOfAddresses = 50;
+ public const int MinimumNumberOfAddresses = 10;
public const int SeedingDelayMs = 1000;
}
diff --git a/src/AddressData.Core/Services/OverpassTurboService.cs b/src/AddressData.Core/Services/OverpassTurboService.cs
index a8f708c..b08f4bb 100644
--- a/src/AddressData.Core/Services/OverpassTurboService.cs
+++ b/src/AddressData.Core/Services/OverpassTurboService.cs
@@ -72,7 +72,8 @@ public class OverpassTurboService
{
var response = await httpClientFactory
.CreateClient()
- .GetAsync($"{Constants.OverpassTurboUrl}{query}");
+ .PostAsync(Constants.OverpassTurboUrl,
+ new FormUrlEncodedContent([new("data", query)]));
await using var stream = await response.Content.ReadAsStreamAsync();
using var streamReader = new StreamReader(stream);
@@ -94,7 +95,8 @@ public class OverpassTurboService
{
var response = await httpClientFactory
.CreateClient()
- .GetAsync($"{Constants.OverpassTurboUrl}{query}");
+ .PostAsync(Constants.OverpassTurboUrl,
+ new FormUrlEncodedContent([new("data", query)]));
await using var stream = await response.Content.ReadAsStreamAsync();
using var streamReader = new StreamReader(stream);
@@ -117,7 +119,8 @@ public class OverpassTurboService
{
var httpResponse = await httpClientFactory
.CreateClient()
- .GetAsync($"{Constants.OverpassTurboUrl}{Constants.OverpassTurboAreaInfoQuery(location.Latitude, location.Longitude)}");
+ .PostAsync(Constants.OverpassTurboUrl,
+ new FormUrlEncodedContent([new("data", Constants.OverpassTurboAreaInfoQuery(location.Latitude, location.Longitude))]));
using var jsonDocument = JsonDocument.Parse(await httpResponse.Content.ReadAsStringAsync());
diff --git a/src/AddressData.WebApi/Dependency/ServiceRegistrations.cs b/src/AddressData.WebApi/Dependency/ServiceRegistrations.cs
index dfa134b..7b312c4 100644
--- a/src/AddressData.WebApi/Dependency/ServiceRegistrations.cs
+++ b/src/AddressData.WebApi/Dependency/ServiceRegistrations.cs
@@ -1,9 +1,10 @@
namespace AddressData.WebApi.Dependency;
using System.Net;
+using Core;
using Core.Services;
using Core.Services.Interfaces;
-using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.AspNetCore.Http.Timeouts;
using Microsoft.OpenApi;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
@@ -21,6 +22,10 @@ public static WebApplicationBuilder ConfigureServices(this WebApplicationBuilder
{
clientBuilder.SetHandlerLifetime(TimeSpan.FromMinutes(5)); //Set lifetime to five minutes
clientBuilder.AddPolicyHandler(GetRetryPolicy());
+ clientBuilder.ConfigureHttpClient(client =>
+ {
+ client.DefaultRequestHeaders.Add("User-Agent", Constants.OverpassTurboUserAgent);
+ });
});
// Configure DI
@@ -65,14 +70,11 @@ public static WebApplicationBuilder AddOpenTelemetryLogs(this WebApplicationBuil
public static WebApplicationBuilder AddLargeApiTimeout(this WebApplicationBuilder builder)
{
- var largeTimeout = TimeSpan.FromDays(10);
-
- builder.Services.Configure(options =>
+ builder.Services.AddRequestTimeouts(options =>
{
- options.Limits.KeepAliveTimeout = largeTimeout;
- options.Limits.RequestHeadersTimeout = largeTimeout;
+ options.DefaultPolicy =
+ new RequestTimeoutPolicy { Timeout = TimeSpan.FromDays(10) };
});
-
return builder;
}
@@ -82,6 +84,7 @@ private static Polly.Retry.AsyncRetryPolicy GetRetryPolicy(
HttpPolicyExtensions
.HandleTransientHttpError()
.OrResult(msg => msg.StatusCode == HttpStatusCode.NotFound)
+ .OrResult(msg => msg.StatusCode == HttpStatusCode.TooManyRequests)
.WaitAndRetryAsync(6, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2,
retryAttempt)));
}
diff --git a/tests/AddressData.FunctionalTests/Features/DocumentsController.feature b/tests/AddressData.FunctionalTests/Features/DocumentsController.feature
index 5c7f625..3bffabe 100644
--- a/tests/AddressData.FunctionalTests/Features/DocumentsController.feature
+++ b/tests/AddressData.FunctionalTests/Features/DocumentsController.feature
@@ -23,7 +23,7 @@
Examples:
| limit |
- | 5 |
+ | 1 |
@InsertCity
Scenario: Insert city should return created result
diff --git a/tests/AddressData.UnitTests/Helpers/FakeHttpMessageHandler.cs b/tests/AddressData.UnitTests/Helpers/FakeHttpMessageHandler.cs
index 9153b2f..a384788 100644
--- a/tests/AddressData.UnitTests/Helpers/FakeHttpMessageHandler.cs
+++ b/tests/AddressData.UnitTests/Helpers/FakeHttpMessageHandler.cs
@@ -4,8 +4,12 @@ namespace AddressData.UnitTests.Helpers;
public class FakeHttpMessageHandler(Queue responses) : HttpMessageHandler
{
+ public HttpRequestMessage? LastRequest { get; private set; }
+
protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
+ LastRequest = request;
+
if (responses.Count > 0)
{
return Task.FromResult(responses.Dequeue());
diff --git a/tests/AddressData.UnitTests/Helpers/HttpClientHelper.cs b/tests/AddressData.UnitTests/Helpers/HttpClientHelper.cs
index a53df61..20cd6ed 100644
--- a/tests/AddressData.UnitTests/Helpers/HttpClientHelper.cs
+++ b/tests/AddressData.UnitTests/Helpers/HttpClientHelper.cs
@@ -18,6 +18,19 @@ public static HttpClient CreateHttpClient(Queue responses)
return new HttpClient(handler);
}
+ public static (HttpClient Client, FakeHttpMessageHandler Handler) CreateHttpClientWithHandler(string content, string mediaType)
+ {
+ var response = CreateHttpResponse(content, mediaType);
+ var handler = new FakeHttpMessageHandler(new Queue([response]));
+ return (new HttpClient(handler), handler);
+ }
+
+ public static (HttpClient Client, FakeHttpMessageHandler Handler) CreateHttpClientWithHandler(Queue responses)
+ {
+ var handler = new FakeHttpMessageHandler(responses);
+ return (new HttpClient(handler), handler);
+ }
+
public static HttpResponseMessage CreateHttpResponse(string content, string mediaType) => new(HttpStatusCode.OK)
{
Content = new StringContent(content, Encoding.UTF8, mediaType)
diff --git a/tests/AddressData.UnitTests/Services/OverpassTurboServiceTests.cs b/tests/AddressData.UnitTests/Services/OverpassTurboServiceTests.cs
index bf8b2e8..045495c 100644
--- a/tests/AddressData.UnitTests/Services/OverpassTurboServiceTests.cs
+++ b/tests/AddressData.UnitTests/Services/OverpassTurboServiceTests.cs
@@ -1,5 +1,6 @@
namespace AddressData.UnitTests.Services;
+using AddressData.Core;
using AddressData.Core.Services;
using AddressData.UnitTests.Helpers;
using Microsoft.Extensions.Logging;
@@ -233,4 +234,105 @@ public async Task GetLocationReturnsNullIfCityIsNull()
Assert.That(locationResult, Is.Null);
}
+
+ [Test]
+ public async Task GetCitiesSendsPostRequest()
+ {
+ var csvContent = "@id,name,name:en\r\n10,TestCity,\r\n";
+ var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(csvContent, "text/csv");
+ _httpClientFactoryMock
+ .Setup(f => f.CreateClient(It.IsAny()))
+ .Returns(httpClient);
+
+ var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object);
+ var _ = await service.GetCities();
+
+ Assert.That(handler.LastRequest, Is.Not.Null);
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(handler.LastRequest!.Method, Is.EqualTo(HttpMethod.Post));
+ Assert.That(handler.LastRequest!.RequestUri, Is.EqualTo(new Uri(Constants.OverpassTurboUrl)));
+ }
+
+ var body = await handler.LastRequest!.Content!.ReadAsStringAsync();
+ Assert.That(body, Is.EqualTo("data=" + Uri.EscapeDataString(Constants.OverpassTurboGetAllCitiesQuery).Replace("%20", "+")));
+ }
+
+ [Test]
+ public async Task GetCitySendsPostRequestWithCorrectQuery()
+ {
+ var csvContent = "@id,name,name:en\r\n123,TestCity,\r\n";
+ var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(csvContent, "text/csv");
+ _httpClientFactoryMock
+ .Setup(f => f.CreateClient(It.IsAny()))
+ .Returns(httpClient);
+
+ var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object);
+ var _ = await service.GetCity(123);
+
+ var body = await handler.LastRequest!.Content!.ReadAsStringAsync();
+ Assert.That(body, Is.EqualTo("data=" + Uri.EscapeDataString(Constants.OverpassTurboGetCityQuery(123)).Replace("%20", "+")));
+ }
+
+ [Test]
+ public async Task GetAddressesSendsPostRequest()
+ {
+ var csvContent = "@lat,@lon,addr:housenumber,addr:street,addr:postcode\r\n";
+ var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(csvContent, "text/csv");
+ _httpClientFactoryMock
+ .Setup(f => f.CreateClient(It.IsAny()))
+ .Returns(httpClient);
+
+ var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object);
+ var _ = await service.GetAddresses(456);
+
+ Assert.That(handler.LastRequest!.Method, Is.EqualTo(HttpMethod.Post));
+
+ var body = await handler.LastRequest!.Content!.ReadAsStringAsync();
+ Assert.That(body, Is.EqualTo("data=" + Uri.EscapeDataString(Constants.OverpassTurboGetAddressesQuery(456)).Replace("%20", "+")));
+ }
+
+ [Test]
+ public async Task GetLocationUsesPostForAllRequests()
+ {
+ var responses = new Queue();
+
+ // 1) lat/long CSV
+ var latLongCsv = "@lat,@lon\r\n12.3456,-98.7654\r\n";
+ responses.Enqueue(HttpClientHelper.CreateHttpResponse(latLongCsv, "text/csv"));
+
+ // 2) JSON state/country
+ var json = /*lang=json,strict*/ @"{
+ ""elements"": [
+ {
+ ""tags"": {
+ ""admin_level"": ""4"",
+ ""name:en"": ""TestState""
+ }
+ },
+ {
+ ""tags"": {
+ ""admin_level"": ""2"",
+ ""name:en"": ""TestCountry""
+ }
+ }
+ ]
+ }";
+ responses.Enqueue(HttpClientHelper.CreateHttpResponse(json, "application/json"));
+
+ // 3) city CSV
+ var cityCsv = "@id,name,name:en\r\n789,TestCity,\r\n";
+ responses.Enqueue(HttpClientHelper.CreateHttpResponse(cityCsv, "text/csv"));
+
+ var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(responses);
+ _httpClientFactoryMock
+ .Setup(f => f.CreateClient(It.IsAny()))
+ .Returns(httpClient);
+
+ var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object);
+ var _ = await service.GetLocation(789);
+
+ Assert.That(handler.LastRequest, Is.Not.Null);
+ Assert.That(handler.LastRequest!.Method, Is.EqualTo(HttpMethod.Post));
+ }
}