diff --git a/src/AddressData.Core/Constants.cs b/src/AddressData.Core/Constants.cs index 009064e..84eef2f 100644 --- a/src/AddressData.Core/Constants.cs +++ b/src/AddressData.Core/Constants.cs @@ -7,16 +7,21 @@ public static class Constants /// Retrieved from https://wiki.openstreetmap.org/wiki/Overpass_API#Public_Overpass_API_instances /// public const string OverpassTurboUrl = "https://overpass-api.de/api/interpreter"; + + /// + /// User Agent is expected as per https://wiki.openstreetmap.org/wiki/Overpass_API#Public_Overpass_API_instances + /// + public const string OverpassTurboUserAgent = "AddressData/1.0"; public const string OverpassTurboGetAllCitiesQuery = - "?data=[out:csv(::id,\"name\",\"name:en\";true;\",\")];area[place=\"city\"];out;"; + "[out:csv(::id,\"name\",\"name:en\";true;\",\")];area[place=\"city\"];out;"; public static string OverpassTurboGetCityQuery(long areaId) => - $"?data=[out:csv(::id,\"name\",\"name:en\";true;\",\")];area({areaId});out;"; + $"[out:csv(::id,\"name\",\"name:en\";true;\",\")];area({areaId});out;"; public static string OverpassTurboGetLatitudeLongitudeQuery(long areaId) => - $"?data=[out:csv(::lat, ::lon;true;\",\")];area({areaId})->.a;node(area.a);out 1;"; + $"[out:csv(::lat, ::lon;true;\",\")];area({areaId})->.a;node(area.a);out 1;"; public static string OverpassTurboGetAddressesQuery(long areaId) => - $"?data=[out:csv(::lat, ::lon, \"addr:housenumber\", \"addr:street\", \"addr:postcode\";true;\",\")];area({areaId});nwr(area)[\"addr:housenumber\"][\"addr:street\"][\"addr:postcode\"];out center;"; + $"[out:csv(::lat, ::lon, \"addr:housenumber\", \"addr:street\", \"addr:postcode\";true;\",\")];area({areaId});nwr(area)[\"addr:housenumber\"][\"addr:street\"][\"addr:postcode\"];out center;"; public static string OverpassTurboAreaInfoQuery(string latitude, string longitude) => - $"?data=[out:json];is_in({latitude},{longitude})->.a;area.a[name][boundary=administrative][admin_level=2];out tags;area.a[name][boundary=administrative][admin_level=4];out tags;"; + $"[out:json];is_in({latitude},{longitude})->.a;area.a[name][boundary=administrative][admin_level=2];out tags;area.a[name][boundary=administrative][admin_level=4];out tags;"; // Overpass Turbo Response public const string OverpassTurboResponseElements = "elements"; @@ -33,6 +38,6 @@ public static string OverpassTurboAreaInfoQuery(string latitude, string longitud public const string ErrorControllerRoute = "error"; // Other settings - public const int MinimumNumberOfAddresses = 50; + public const int MinimumNumberOfAddresses = 10; public const int SeedingDelayMs = 1000; } diff --git a/src/AddressData.Core/Services/OverpassTurboService.cs b/src/AddressData.Core/Services/OverpassTurboService.cs index a8f708c..b08f4bb 100644 --- a/src/AddressData.Core/Services/OverpassTurboService.cs +++ b/src/AddressData.Core/Services/OverpassTurboService.cs @@ -72,7 +72,8 @@ public class OverpassTurboService { var response = await httpClientFactory .CreateClient() - .GetAsync($"{Constants.OverpassTurboUrl}{query}"); + .PostAsync(Constants.OverpassTurboUrl, + new FormUrlEncodedContent([new("data", query)])); await using var stream = await response.Content.ReadAsStreamAsync(); using var streamReader = new StreamReader(stream); @@ -94,7 +95,8 @@ public class OverpassTurboService { var response = await httpClientFactory .CreateClient() - .GetAsync($"{Constants.OverpassTurboUrl}{query}"); + .PostAsync(Constants.OverpassTurboUrl, + new FormUrlEncodedContent([new("data", query)])); await using var stream = await response.Content.ReadAsStreamAsync(); using var streamReader = new StreamReader(stream); @@ -117,7 +119,8 @@ public class OverpassTurboService { var httpResponse = await httpClientFactory .CreateClient() - .GetAsync($"{Constants.OverpassTurboUrl}{Constants.OverpassTurboAreaInfoQuery(location.Latitude, location.Longitude)}"); + .PostAsync(Constants.OverpassTurboUrl, + new FormUrlEncodedContent([new("data", Constants.OverpassTurboAreaInfoQuery(location.Latitude, location.Longitude))])); using var jsonDocument = JsonDocument.Parse(await httpResponse.Content.ReadAsStringAsync()); diff --git a/src/AddressData.WebApi/Dependency/ServiceRegistrations.cs b/src/AddressData.WebApi/Dependency/ServiceRegistrations.cs index dfa134b..7b312c4 100644 --- a/src/AddressData.WebApi/Dependency/ServiceRegistrations.cs +++ b/src/AddressData.WebApi/Dependency/ServiceRegistrations.cs @@ -1,9 +1,10 @@ namespace AddressData.WebApi.Dependency; using System.Net; +using Core; using Core.Services; using Core.Services.Interfaces; -using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.AspNetCore.Http.Timeouts; using Microsoft.OpenApi; using OpenTelemetry.Resources; using OpenTelemetry.Trace; @@ -21,6 +22,10 @@ public static WebApplicationBuilder ConfigureServices(this WebApplicationBuilder { clientBuilder.SetHandlerLifetime(TimeSpan.FromMinutes(5)); //Set lifetime to five minutes clientBuilder.AddPolicyHandler(GetRetryPolicy()); + clientBuilder.ConfigureHttpClient(client => + { + client.DefaultRequestHeaders.Add("User-Agent", Constants.OverpassTurboUserAgent); + }); }); // Configure DI @@ -65,14 +70,11 @@ public static WebApplicationBuilder AddOpenTelemetryLogs(this WebApplicationBuil public static WebApplicationBuilder AddLargeApiTimeout(this WebApplicationBuilder builder) { - var largeTimeout = TimeSpan.FromDays(10); - - builder.Services.Configure(options => + builder.Services.AddRequestTimeouts(options => { - options.Limits.KeepAliveTimeout = largeTimeout; - options.Limits.RequestHeadersTimeout = largeTimeout; + options.DefaultPolicy = + new RequestTimeoutPolicy { Timeout = TimeSpan.FromDays(10) }; }); - return builder; } @@ -82,6 +84,7 @@ private static Polly.Retry.AsyncRetryPolicy GetRetryPolicy( HttpPolicyExtensions .HandleTransientHttpError() .OrResult(msg => msg.StatusCode == HttpStatusCode.NotFound) + .OrResult(msg => msg.StatusCode == HttpStatusCode.TooManyRequests) .WaitAndRetryAsync(6, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))); } diff --git a/tests/AddressData.FunctionalTests/Features/DocumentsController.feature b/tests/AddressData.FunctionalTests/Features/DocumentsController.feature index 5c7f625..3bffabe 100644 --- a/tests/AddressData.FunctionalTests/Features/DocumentsController.feature +++ b/tests/AddressData.FunctionalTests/Features/DocumentsController.feature @@ -23,7 +23,7 @@ Examples: | limit | - | 5 | + | 1 | @InsertCity Scenario: Insert city should return created result diff --git a/tests/AddressData.UnitTests/Helpers/FakeHttpMessageHandler.cs b/tests/AddressData.UnitTests/Helpers/FakeHttpMessageHandler.cs index 9153b2f..a384788 100644 --- a/tests/AddressData.UnitTests/Helpers/FakeHttpMessageHandler.cs +++ b/tests/AddressData.UnitTests/Helpers/FakeHttpMessageHandler.cs @@ -4,8 +4,12 @@ namespace AddressData.UnitTests.Helpers; public class FakeHttpMessageHandler(Queue responses) : HttpMessageHandler { + public HttpRequestMessage? LastRequest { get; private set; } + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { + LastRequest = request; + if (responses.Count > 0) { return Task.FromResult(responses.Dequeue()); diff --git a/tests/AddressData.UnitTests/Helpers/HttpClientHelper.cs b/tests/AddressData.UnitTests/Helpers/HttpClientHelper.cs index a53df61..20cd6ed 100644 --- a/tests/AddressData.UnitTests/Helpers/HttpClientHelper.cs +++ b/tests/AddressData.UnitTests/Helpers/HttpClientHelper.cs @@ -18,6 +18,19 @@ public static HttpClient CreateHttpClient(Queue responses) return new HttpClient(handler); } + public static (HttpClient Client, FakeHttpMessageHandler Handler) CreateHttpClientWithHandler(string content, string mediaType) + { + var response = CreateHttpResponse(content, mediaType); + var handler = new FakeHttpMessageHandler(new Queue([response])); + return (new HttpClient(handler), handler); + } + + public static (HttpClient Client, FakeHttpMessageHandler Handler) CreateHttpClientWithHandler(Queue responses) + { + var handler = new FakeHttpMessageHandler(responses); + return (new HttpClient(handler), handler); + } + public static HttpResponseMessage CreateHttpResponse(string content, string mediaType) => new(HttpStatusCode.OK) { Content = new StringContent(content, Encoding.UTF8, mediaType) diff --git a/tests/AddressData.UnitTests/Services/OverpassTurboServiceTests.cs b/tests/AddressData.UnitTests/Services/OverpassTurboServiceTests.cs index bf8b2e8..045495c 100644 --- a/tests/AddressData.UnitTests/Services/OverpassTurboServiceTests.cs +++ b/tests/AddressData.UnitTests/Services/OverpassTurboServiceTests.cs @@ -1,5 +1,6 @@ namespace AddressData.UnitTests.Services; +using AddressData.Core; using AddressData.Core.Services; using AddressData.UnitTests.Helpers; using Microsoft.Extensions.Logging; @@ -233,4 +234,105 @@ public async Task GetLocationReturnsNullIfCityIsNull() Assert.That(locationResult, Is.Null); } + + [Test] + public async Task GetCitiesSendsPostRequest() + { + var csvContent = "@id,name,name:en\r\n10,TestCity,\r\n"; + var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(csvContent, "text/csv"); + _httpClientFactoryMock + .Setup(f => f.CreateClient(It.IsAny())) + .Returns(httpClient); + + var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object); + var _ = await service.GetCities(); + + Assert.That(handler.LastRequest, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(handler.LastRequest!.Method, Is.EqualTo(HttpMethod.Post)); + Assert.That(handler.LastRequest!.RequestUri, Is.EqualTo(new Uri(Constants.OverpassTurboUrl))); + } + + var body = await handler.LastRequest!.Content!.ReadAsStringAsync(); + Assert.That(body, Is.EqualTo("data=" + Uri.EscapeDataString(Constants.OverpassTurboGetAllCitiesQuery).Replace("%20", "+"))); + } + + [Test] + public async Task GetCitySendsPostRequestWithCorrectQuery() + { + var csvContent = "@id,name,name:en\r\n123,TestCity,\r\n"; + var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(csvContent, "text/csv"); + _httpClientFactoryMock + .Setup(f => f.CreateClient(It.IsAny())) + .Returns(httpClient); + + var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object); + var _ = await service.GetCity(123); + + var body = await handler.LastRequest!.Content!.ReadAsStringAsync(); + Assert.That(body, Is.EqualTo("data=" + Uri.EscapeDataString(Constants.OverpassTurboGetCityQuery(123)).Replace("%20", "+"))); + } + + [Test] + public async Task GetAddressesSendsPostRequest() + { + var csvContent = "@lat,@lon,addr:housenumber,addr:street,addr:postcode\r\n"; + var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(csvContent, "text/csv"); + _httpClientFactoryMock + .Setup(f => f.CreateClient(It.IsAny())) + .Returns(httpClient); + + var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object); + var _ = await service.GetAddresses(456); + + Assert.That(handler.LastRequest!.Method, Is.EqualTo(HttpMethod.Post)); + + var body = await handler.LastRequest!.Content!.ReadAsStringAsync(); + Assert.That(body, Is.EqualTo("data=" + Uri.EscapeDataString(Constants.OverpassTurboGetAddressesQuery(456)).Replace("%20", "+"))); + } + + [Test] + public async Task GetLocationUsesPostForAllRequests() + { + var responses = new Queue(); + + // 1) lat/long CSV + var latLongCsv = "@lat,@lon\r\n12.3456,-98.7654\r\n"; + responses.Enqueue(HttpClientHelper.CreateHttpResponse(latLongCsv, "text/csv")); + + // 2) JSON state/country + var json = /*lang=json,strict*/ @"{ + ""elements"": [ + { + ""tags"": { + ""admin_level"": ""4"", + ""name:en"": ""TestState"" + } + }, + { + ""tags"": { + ""admin_level"": ""2"", + ""name:en"": ""TestCountry"" + } + } + ] + }"; + responses.Enqueue(HttpClientHelper.CreateHttpResponse(json, "application/json")); + + // 3) city CSV + var cityCsv = "@id,name,name:en\r\n789,TestCity,\r\n"; + responses.Enqueue(HttpClientHelper.CreateHttpResponse(cityCsv, "text/csv")); + + var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(responses); + _httpClientFactoryMock + .Setup(f => f.CreateClient(It.IsAny())) + .Returns(httpClient); + + var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object); + var _ = await service.GetLocation(789); + + Assert.That(handler.LastRequest, Is.Not.Null); + Assert.That(handler.LastRequest!.Method, Is.EqualTo(HttpMethod.Post)); + } }