fix: align addressdata with new Overpass API contract

src/AddressData.Core/Constants.cs

@@ -7,16 +7,21 @@ public static class Constants
     /// Retrieved from https://wiki.openstreetmap.org/wiki/Overpass_API#Public_Overpass_API_instances
     /// </summary>
     public const string OverpassTurboUrl = "https://overpass-api.de/api/interpreter";
+
+    /// <summary>
+    /// User Agent is expected as per https://wiki.openstreetmap.org/wiki/Overpass_API#Public_Overpass_API_instances
+    /// </summary>
+    public const string OverpassTurboUserAgent = "AddressData/1.0";
     public const string OverpassTurboGetAllCitiesQuery =
-        "?data=[out:csv(::id,\"name\",\"name:en\";true;\",\")];area[place=\"city\"];out;";
+        "[out:csv(::id,\"name\",\"name:en\";true;\",\")];area[place=\"city\"];out;";
     public static string OverpassTurboGetCityQuery(long areaId) =>
-        $"?data=[out:csv(::id,\"name\",\"name:en\";true;\",\")];area({areaId});out;";
+        $"[out:csv(::id,\"name\",\"name:en\";true;\",\")];area({areaId});out;";
     public static string OverpassTurboGetLatitudeLongitudeQuery(long areaId) =>
-        $"?data=[out:csv(::lat, ::lon;true;\",\")];area({areaId})->.a;node(area.a);out 1;";
+        $"[out:csv(::lat, ::lon;true;\",\")];area({areaId})->.a;node(area.a);out 1;";
     public static string OverpassTurboGetAddressesQuery(long areaId) =>
-        $"?data=[out:csv(::lat, ::lon, \"addr:housenumber\", \"addr:street\", \"addr:postcode\";true;\",\")];area({areaId});nwr(area)[\"addr:housenumber\"][\"addr:street\"][\"addr:postcode\"];out center;";
+        $"[out:csv(::lat, ::lon, \"addr:housenumber\", \"addr:street\", \"addr:postcode\";true;\",\")];area({areaId});nwr(area)[\"addr:housenumber\"][\"addr:street\"][\"addr:postcode\"];out center;";
     public static string OverpassTurboAreaInfoQuery(string latitude, string longitude) =>
-        $"?data=[out:json];is_in({latitude},{longitude})->.a;area.a[name][boundary=administrative][admin_level=2];out tags;area.a[name][boundary=administrative][admin_level=4];out tags;";
+        $"[out:json];is_in({latitude},{longitude})->.a;area.a[name][boundary=administrative][admin_level=2];out tags;area.a[name][boundary=administrative][admin_level=4];out tags;";
 
     // Overpass Turbo Response
     public const string OverpassTurboResponseElements = "elements";
@@ -33,6 +38,6 @@ public static string OverpassTurboAreaInfoQuery(string latitude, string longitud
     public const string ErrorControllerRoute = "error";
 
     // Other settings
-    public const int MinimumNumberOfAddresses = 50;
+    public const int MinimumNumberOfAddresses = 10;
     public const int SeedingDelayMs = 1000;
 }

src/AddressData.Core/Services/OverpassTurboService.cs

@@ -72,7 +72,8 @@
         {
             var response = await httpClientFactory
                 .CreateClient()
-                .GetAsync($"{Constants.OverpassTurboUrl}{query}");
+                .PostAsync(Constants.OverpassTurboUrl,
+                    new FormUrlEncodedContent([new("data", query)]));
 
             await using var stream = await response.Content.ReadAsStreamAsync();
             using var streamReader = new StreamReader(stream);
@@ -94,7 +95,8 @@
         {
             var response = await httpClientFactory
                 .CreateClient()
-                .GetAsync($"{Constants.OverpassTurboUrl}{query}");
+                .PostAsync(Constants.OverpassTurboUrl,
+                    new FormUrlEncodedContent([new("data", query)]));
 
             await using var stream = await response.Content.ReadAsStreamAsync();
             using var streamReader = new StreamReader(stream);
@@ -117,7 +119,8 @@
     {
         var httpResponse = await httpClientFactory
             .CreateClient()
-            .GetAsync($"{Constants.OverpassTurboUrl}{Constants.OverpassTurboAreaInfoQuery(location.Latitude, location.Longitude)}");
+            .PostAsync(Constants.OverpassTurboUrl,
+                new FormUrlEncodedContent([new("data", Constants.OverpassTurboAreaInfoQuery(location.Latitude, location.Longitude))]));
 
         using var jsonDocument = JsonDocument.Parse(await httpResponse.Content.ReadAsStringAsync());
 
@@ -126,8 +129,8 @@
 
     private static StateCountryDomainModel? ParseJson(JsonDocument jsonDocument)
     {
         string countryName = null;
         string stateName = null;
 
         // TODO: HACK: Manually parsing JSON here. There must be a better way...
         var elements = jsonDocument.RootElement.GetProperty(Constants.OverpassTurboResponseElements);
@@ -142,22 +145,22 @@
                     {
                         if (tags.TryGetProperty(Constants.OverpassTurboResponseEnglishName, out var stateEnglish))
                         {
                             stateName = stateEnglish.GetString();
                         }
                         else if (tags.TryGetProperty(Constants.OverpassTurboResponseName, out var state))
                         {
                             stateName = state.GetString();
                         }
                     }
                     else if (adminLevel.GetString() == Constants.OverpassTurboResponseCountryAdministrationLevel)
                     {
                         if (tags.TryGetProperty(Constants.OverpassTurboResponseEnglishName, out var countryEnglish))
                         {
                             countryName = countryEnglish.GetString();
                         }
                         else if (tags.TryGetProperty(Constants.OverpassTurboResponseName, out var country))
                         {
                             countryName = country.GetString();
                         }
                     }
                 }

src/AddressData.WebApi/Dependency/ServiceRegistrations.cs

@@ -1,9 +1,10 @@
 namespace AddressData.WebApi.Dependency;
 
 using System.Net;
+using Core;
 using Core.Services;
 using Core.Services.Interfaces;
-using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.AspNetCore.Http.Timeouts;
 using Microsoft.OpenApi;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
@@ -21,6 +22,10 @@ public static WebApplicationBuilder ConfigureServices(this WebApplicationBuilder
             {
                 clientBuilder.SetHandlerLifetime(TimeSpan.FromMinutes(5)); //Set lifetime to five minutes
                 clientBuilder.AddPolicyHandler(GetRetryPolicy());
+                clientBuilder.ConfigureHttpClient(client =>
+                {
+                    client.DefaultRequestHeaders.Add("User-Agent", Constants.OverpassTurboUserAgent);
+                });
             });
 
         // Configure DI
@@ -65,14 +70,11 @@ public static WebApplicationBuilder AddOpenTelemetryLogs(this WebApplicationBuil
 
     public static WebApplicationBuilder AddLargeApiTimeout(this WebApplicationBuilder builder)
     {
-        var largeTimeout = TimeSpan.FromDays(10);
-
-        builder.Services.Configure<KestrelServerOptions>(options =>
+        builder.Services.AddRequestTimeouts(options =>
         {
-            options.Limits.KeepAliveTimeout = largeTimeout;
-            options.Limits.RequestHeadersTimeout = largeTimeout;
+            options.DefaultPolicy =
+                new RequestTimeoutPolicy { Timeout = TimeSpan.FromDays(10) };
         });
-
         return builder;
     }
 
@@ -82,6 +84,7 @@ private static Polly.Retry.AsyncRetryPolicy<HttpResponseMessage> GetRetryPolicy(
         HttpPolicyExtensions
             .HandleTransientHttpError()
             .OrResult(msg => msg.StatusCode == HttpStatusCode.NotFound)
+            .OrResult(msg => msg.StatusCode == HttpStatusCode.TooManyRequests)
             .WaitAndRetryAsync(6, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2,
                 retryAttempt)));
 }

tests/AddressData.FunctionalTests/Features/DocumentsController.feature

@@ -23,7 +23,7 @@
 
     Examples:
       | limit |
-      | 5     |
+      | 1     |
 
   @InsertCity
   Scenario: Insert city should return created result

tests/AddressData.UnitTests/Helpers/FakeHttpMessageHandler.cs

@@ -4,8 +4,12 @@ namespace AddressData.UnitTests.Helpers;
 
 public class FakeHttpMessageHandler(Queue<HttpResponseMessage> responses) : HttpMessageHandler
 {
+    public HttpRequestMessage? LastRequest { get; private set; }
+
     protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
     {
+        LastRequest = request;
+
         if (responses.Count > 0)
         {
             return Task.FromResult(responses.Dequeue());

tests/AddressData.UnitTests/Helpers/HttpClientHelper.cs

@@ -18,6 +18,19 @@ public static HttpClient CreateHttpClient(Queue<HttpResponseMessage> responses)
         return new HttpClient(handler);
     }
 
+    public static (HttpClient Client, FakeHttpMessageHandler Handler) CreateHttpClientWithHandler(string content, string mediaType)
+    {
+        var response = CreateHttpResponse(content, mediaType);
+        var handler = new FakeHttpMessageHandler(new Queue<HttpResponseMessage>([response]));
+        return (new HttpClient(handler), handler);
+    }
+
+    public static (HttpClient Client, FakeHttpMessageHandler Handler) CreateHttpClientWithHandler(Queue<HttpResponseMessage> responses)
+    {
+        var handler = new FakeHttpMessageHandler(responses);
+        return (new HttpClient(handler), handler);
+    }
+
     public static HttpResponseMessage CreateHttpResponse(string content, string mediaType) => new(HttpStatusCode.OK)
     {
         Content = new StringContent(content, Encoding.UTF8, mediaType)

tests/AddressData.UnitTests/Services/OverpassTurboServiceTests.cs

@@ -1,5 +1,6 @@
 namespace AddressData.UnitTests.Services;
 
+using AddressData.Core;
 using AddressData.Core.Services;
 using AddressData.UnitTests.Helpers;
 using Microsoft.Extensions.Logging;
@@ -233,4 +234,105 @@ public async Task GetLocationReturnsNullIfCityIsNull()
 
         Assert.That(locationResult, Is.Null);
     }
+
+    [Test]
+    public async Task GetCitiesSendsPostRequest()
+    {
+        var csvContent = "@id,name,name:en\r\n10,TestCity,\r\n";
+        var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(csvContent, "text/csv");
+        _httpClientFactoryMock
+            .Setup(f => f.CreateClient(It.IsAny<string>()))
+            .Returns(httpClient);
+
+        var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object);
+        var _ = await service.GetCities();
+
+        Assert.That(handler.LastRequest, Is.Not.Null);
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(handler.LastRequest!.Method, Is.EqualTo(HttpMethod.Post));
+            Assert.That(handler.LastRequest!.RequestUri, Is.EqualTo(new Uri(Constants.OverpassTurboUrl)));
+        }
+
+        var body = await handler.LastRequest!.Content!.ReadAsStringAsync();
+        Assert.That(body, Is.EqualTo("data=" + Uri.EscapeDataString(Constants.OverpassTurboGetAllCitiesQuery).Replace("%20", "+")));
+    }
+
+    [Test]
+    public async Task GetCitySendsPostRequestWithCorrectQuery()
+    {
+        var csvContent = "@id,name,name:en\r\n123,TestCity,\r\n";
+        var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(csvContent, "text/csv");
+        _httpClientFactoryMock
+            .Setup(f => f.CreateClient(It.IsAny<string>()))
+            .Returns(httpClient);
+
+        var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object);
+        var _ = await service.GetCity(123);
+
+        var body = await handler.LastRequest!.Content!.ReadAsStringAsync();
+        Assert.That(body, Is.EqualTo("data=" + Uri.EscapeDataString(Constants.OverpassTurboGetCityQuery(123)).Replace("%20", "+")));
+    }
+
+    [Test]
+    public async Task GetAddressesSendsPostRequest()
+    {
+        var csvContent = "@lat,@lon,addr:housenumber,addr:street,addr:postcode\r\n";
+        var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(csvContent, "text/csv");
+        _httpClientFactoryMock
+            .Setup(f => f.CreateClient(It.IsAny<string>()))
+            .Returns(httpClient);
+
+        var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object);
+        var _ = await service.GetAddresses(456);
+
+        Assert.That(handler.LastRequest!.Method, Is.EqualTo(HttpMethod.Post));
+
+        var body = await handler.LastRequest!.Content!.ReadAsStringAsync();
+        Assert.That(body, Is.EqualTo("data=" + Uri.EscapeDataString(Constants.OverpassTurboGetAddressesQuery(456)).Replace("%20", "+")));
+    }
+
+    [Test]
+    public async Task GetLocationUsesPostForAllRequests()
+    {
+        var responses = new Queue<HttpResponseMessage>();
+
+        // 1) lat/long CSV
+        var latLongCsv = "@lat,@lon\r\n12.3456,-98.7654\r\n";
+        responses.Enqueue(HttpClientHelper.CreateHttpResponse(latLongCsv, "text/csv"));
+
+        // 2) JSON state/country
+        var json = /*lang=json,strict*/ @"{
+                ""elements"": [
+                    {
+                        ""tags"": {
+                            ""admin_level"": ""4"",
+                            ""name:en"": ""TestState""
+                        }
+                    },
+                    {
+                        ""tags"": {
+                            ""admin_level"": ""2"",
+                            ""name:en"": ""TestCountry""
+                        }
+                    }
+                ]
+            }";
+        responses.Enqueue(HttpClientHelper.CreateHttpResponse(json, "application/json"));
+
+        // 3) city CSV
+        var cityCsv = "@id,name,name:en\r\n789,TestCity,\r\n";
+        responses.Enqueue(HttpClientHelper.CreateHttpResponse(cityCsv, "text/csv"));
+
+        var (httpClient, handler) = HttpClientHelper.CreateHttpClientWithHandler(responses);
+        _httpClientFactoryMock
+            .Setup(f => f.CreateClient(It.IsAny<string>()))
+            .Returns(httpClient);
+
+        var service = new OverpassTurboService(_httpClientFactoryMock.Object, _loggerMock.Object);
+        var _ = await service.GetLocation(789);
+
+        Assert.That(handler.LastRequest, Is.Not.Null);
+        Assert.That(handler.LastRequest!.Method, Is.EqualTo(HttpMethod.Post));
+    }
 }