Split test server infrastructure: TestServer for HttpClient, Kestrel for PuppeteerSharp

Directory.Packages.props

@@ -9,7 +9,8 @@
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
+    <PackageVersion Include="PuppeteerSharp" Version="20.2.5" />
     <PackageVersion Include="xunit" Version="2.9.2" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
   </ItemGroup>
-</Project>
+</Project>

NetInteractor.sln

@@ -9,6 +9,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetInteractor", "src\NetInt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetInteractor.Test", "test\NetInteractor.Test\NetInteractor.Test.csproj", "{A9186BD7-EFFC-4B4D-B828-C4F4AA924C50}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetInteractor.PuppeteerSharp", "src\NetInteractor.PuppeteerSharp\NetInteractor.PuppeteerSharp.csproj", "{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -18,9 +20,6 @@ Global
 		Release|x64 = Release|x64
 		Release|x86 = Release|x86
 	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{75781078-1761-4EDD-A872-5F71D1FF9721}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{75781078-1761-4EDD-A872-5F71D1FF9721}.Debug|Any CPU.Build.0 = Debug|Any CPU
@@ -46,9 +45,25 @@ Global
 		{A9186BD7-EFFC-4B4D-B828-C4F4AA924C50}.Release|x64.Build.0 = Release|Any CPU
 		{A9186BD7-EFFC-4B4D-B828-C4F4AA924C50}.Release|x86.ActiveCfg = Release|Any CPU
 		{A9186BD7-EFFC-4B4D-B828-C4F4AA924C50}.Release|x86.Build.0 = Release|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Debug|x64.Build.0 = Debug|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Debug|x86.Build.0 = Debug|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Release|x64.ActiveCfg = Release|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Release|x64.Build.0 = Release|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Release|x86.ActiveCfg = Release|Any CPU
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Release|x86.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{75781078-1761-4EDD-A872-5F71D1FF9721} = {F5108642-5386-4F95-ACF8-8DA0AD621820}
 		{A9186BD7-EFFC-4B4D-B828-C4F4AA924C50} = {F5108642-5386-4F95-ACF8-8DA0AD621820}
+		{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF} = {F5108642-5386-4F95-ACF8-8DA0AD621820}
 	EndGlobalSection
 EndGlobal

src/NetInteractor.PuppeteerSharp/NetInteractor.PuppeteerSharp.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.0;net8.0;net9.0</TargetFrameworks>
+    <Authors>Kerry Jiang</Authors>
+    <Description>PuppeteerSharp implementation of IWebAccessor for NetInteractor - enables browser automation with headless Chrome/Chromium for advanced web interactions.</Description>
+    <PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
+    <PackageTags>Web;Automation;Html;Scraping;Testing;PuppeteerSharp;Browser;WebAutomation;Headless;Chrome;Chromium</PackageTags>
+    <RepositoryUrl>https://github.com/kerryjiang/NetInteractor.git</RepositoryUrl>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(IncludeReleaseNotes)' == 'true'">
+    <PackageReadmeFile>v$(PackageVersion).md</PackageReadmeFile>
+  </PropertyGroup>
+  <ItemGroup Condition="'$(IncludeReleaseNotes)' == 'true'">
+    <None Include="../../releaseNotes/v$(PackageVersion).md" Pack="true" PackagePath="/" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="../NetInteractor/NetInteractor.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="PuppeteerSharp" />
+  </ItemGroup>
+</Project>

src/NetInteractor.PuppeteerSharp/PuppeteerSharpWebAccessor.cs

@@ -0,0 +1,227 @@
+using System;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Threading;
+using System.Threading.Tasks;
+using PuppeteerSharp;
+
+namespace NetInteractor.WebAccessors
+{
+    public class PuppeteerSharpWebAccessor : IWebAccessor, IDisposable
+    {
+        private IBrowser _browser;
+        private readonly LaunchOptions _launchOptions;
+        private readonly SemaphoreSlim _browserLock = new SemaphoreSlim(1, 1);
+        private bool _disposed;
+
+        public PuppeteerSharpWebAccessor(LaunchOptions launchOptions = null)
+        {
+            _launchOptions = launchOptions ?? new LaunchOptions
+            {
+                Headless = true,
+                Args = new[] { "--no-sandbox", "--disable-setuid-sandbox" }
+            };
+        }
+
+        private async Task<IBrowser> GetBrowserAsync()
+        {
+            if (_browser == null)
+            {
+                await _browserLock.WaitAsync();
+                try
+                {
+                    // Double-check after acquiring the lock
+                    if (_browser == null)
+                    {
+                        // Download browser if needed
+                        var browserFetcher = new BrowserFetcher();
+
+                        // Download the browser (it will skip if already downloaded)
+                        await browserFetcher.DownloadAsync();
+
+                        _browser = await Puppeteer.LaunchAsync(_launchOptions);
+                    }
+                }
+                finally
+                {
+                    _browserLock.Release();
+                }
+            }
+            return _browser;
+        }
+
+        public async Task<ResponseInfo> GetAsync(string url)
+        {
+            var browser = await GetBrowserAsync();
+            var page = await browser.NewPageAsync();
+
+            try
+            {
+                var response = await page.GoToAsync(url, new NavigationOptions
+                {
+                    WaitUntil = new[] { WaitUntilNavigation.Networkidle0 }
+                });
+
+                return await GetResultFromResponse(page, response);
+            }
+            finally
+            {
+                await page.CloseAsync();
+            }
+        }
+
+        public async Task<ResponseInfo> PostAsync(string url, NameValueCollection formValues)
+        {
+            var browser = await GetBrowserAsync();
+            var page = await browser.NewPageAsync();
+
+            try
+            {
+                // Navigate to the page first
+                await page.GoToAsync(url, new NavigationOptions
+                {
+                    WaitUntil = new[] { WaitUntilNavigation.Networkidle0 }
+                });
+
+                // Set up request interception to modify the request to POST
+                await page.SetRequestInterceptionAsync(true);
+
+                var formData = string.Join("&", formValues.Keys.OfType<string>()
+                    .Select(k => k + "=" + Uri.EscapeDataString(formValues[k])));
+
+                var responseTaskSource = new TaskCompletionSource<IResponse>(TaskCreationOptions.RunContinuationsAsynchronously);
+                var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+
+                EventHandler<RequestEventArgs> requestHandler = null;
+                EventHandler<ResponseCreatedEventArgs> responseHandler = null;
+
+                try
+                {
+                    requestHandler = async (sender, e) =>
+                    {
+                        if (e.Request.Url == url)
+                        {
+                            await e.Request.ContinueAsync(new Payload
+                            {
+                                Method = HttpMethod.Post,
+                                PostData = formData,
+                                Headers = e.Request.Headers.Concat(new[]
+                                {
+                                    new System.Collections.Generic.KeyValuePair<string, string>(
+                                        "Content-Type", "application/x-www-form-urlencoded")
+                                }).ToDictionary(kvp => kvp.Key, kvp => kvp.Value)
+                            });
+                        }
+                        else
+                        {
+                            await e.Request.ContinueAsync();
+                        }
+                    };
+
+                    responseHandler = (sender, e) =>
+                    {
+                        if (e.Response.Url == url)
+                        {
+                            responseTaskSource.TrySetResult(e.Response);
+                        }
+                    };
+
+                    page.Request += requestHandler;
+                    page.Response += responseHandler;
+
+                    // Register cancellation
+                    cts.Token.Register(() => responseTaskSource.TrySetCanceled());
+
+                    // Navigate again to trigger the POST
+                    await page.GoToAsync(url, new NavigationOptions
+                    {
+                        WaitUntil = new[] { WaitUntilNavigation.Networkidle0 }
+                    });
+
+                    var response = await responseTaskSource.Task;
+                    return await GetResultFromResponse(page, response);
+                }
+                finally
+                {
+                    // Clean up event handlers
+                    if (requestHandler != null)
+                        page.Request -= requestHandler;
+                    if (responseHandler != null)
+                        page.Response -= responseHandler;
+                    cts.Dispose();
+                }
+            }
+            finally
+            {
+                await page.CloseAsync();
+            }
+        }
+
+        private async Task<ResponseInfo> GetResultFromResponse(IPage page, IResponse response)
+        {
+            var html = await page.GetContentAsync();
+
+            // Create a mock HttpResponseHeaders since we cannot instantiate it directly
+            // We'll store headers in a custom wrapper that mimics HttpResponseHeaders
+            var mockHeaders = CreateMockHeaders(response.Headers);
+
+            return new ResponseInfo
+            {
+                StatusCode = (int)response.Status,
+                StatusDescription = response.StatusText,
+                Html = html,
+                Url = response.Url,
+                Headers = mockHeaders
+            };
+        }
+
+        private HttpResponseHeaders CreateMockHeaders(System.Collections.Generic.IDictionary<string, string> headers)
+        {
+            // Since HttpResponseHeaders is sealed and cannot be instantiated directly,
+            // we create it via HttpResponseMessage which is the standard way
+            var mockResponse = new System.Net.Http.HttpResponseMessage();
+
+            if (headers != null)
+            {
+                foreach (var header in headers)
+                {
+                    try
+                    {
+                        mockResponse.Headers.TryAddWithoutValidation(header.Key, header.Value);
+                    }
+                    catch (ArgumentException)
+                    {
+                        // Some headers might be content headers, skip them
+                    }
+                    catch (FormatException)
+                    {
+                        // Invalid header format, skip it
+                    }
+                }
+            }
+
+            return mockResponse.Headers;
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!_disposed)
+            {
+                if (disposing)
+                {
+                    _browser?.Dispose();
+                    _browserLock?.Dispose();
+                }
+                _disposed = true;
+            }
+        }
+    }
+}