使用线程池限制最大线程数 – 尝试读取或写入受保护的内存错误
我正在使用Noseratio的一些报废代码，请访问https://stackoverflow.com/a/22262976/3499115 。 他写了它来刮取一个url列表，但是我正在使用它，但是在我正在使用的另一个网络爬虫MVC控制器中一次只渲染一个url。 每当我找到特定类型的链接时，我都会调用此代码，并且看起来多次这样做会导致内存不足。 也许解决方案是使用线程池并限制最大线程数，但我将如何对此代码执行此操作？ 以下是调用webbrowser代码的Web爬网程序代码：
public static HtmlDocument renderJavascript(string url) { HtmlDocument doc = new HtmlDocument(); // using webBrowserScrapper try { WebBrowserExt.SetFeatureBrowserEmulation(); // enable HTML5 var cts = new CancellationTokenSource((int)TimeSpan.FromMinutes(3).TotalMilliseconds); var task = WebBrowserScrapper.ScrapSitesAsync( url, cts.Token); task.Wait(); //Console.WriteLine("Press Enter to exit..."); //Console.ReadLine(); doc.LoadHtml(task.Result); return doc; } catch (Exception ex) { while (ex is AggregateException && ex.InnerException != null) ex = ex.InnerException; Console.WriteLine(ex.Message); //Environment.Exit(-1); } return null; } webbrowser代码（我刚刚将参数更改为ScrapSitesAsync函数中的单个字符串：
using System; using using System.Linq; using System.Text; using Microsoft.Win32; using System.Threading; using System.Threading.Tasks; using System.Windows.Forms; namespace Abot.Demo { public class WebBrowserScrapper { // by Noseratio - https://stackoverflow.com/a/22262976/1768303 // main logic public static async Task ScrapSitesAsync(string url, CancellationToken token) { using (var apartment = new MessageLoopApartment()) { // create WebBrowser inside MessageLoopApartment var webBrowser = apartment.Invoke(() => new WebBrowser()); try { Console.WriteLine("WebBrowser URL:\n" + url); // cancel in 30s or when the main token is signalled var navigationCts = CancellationTokenSource.CreateLinkedTokenSource(token); navigationCts.CancelAfter((int)TimeSpan.FromSeconds(10).TotalMilliseconds); var navigationToken = navigationCts.Token; // run the navigation task inside MessageLoopApartment string html = await apartment.Run(() => webBrowser.NavigateAsync(url, navigationToken), navigationToken); Console.WriteLine("Scrape complete for URL:\n" + url); return html; } finally { // dispose of WebBrowser inside MessageLoopApartment apartment.Invoke(() => webBrowser.Dispose()); } } } } /// /// WebBrowserExt - WebBrowser extensions /// by Noseratio - https://stackoverflow.com/a/22262976/1768303 /// public static class WebBrowserExt { const int POLL_DELAY = 500; // navigate and download public static async Task NavigateAsync(this WebBrowser webBrowser, string url, CancellationToken token) { // navigate and await DocumentCompleted var tcs = new TaskCompletionSource(); WebBrowserDocumentCompletedEventHandler handler = (s, arg) => tcs.TrySetResult(true); using (token.Register( () => { webBrowser.Stop(); tcs.TrySetCanceled(); }, useSynchronizationContext: true)) { webBrowser.DocumentCompleted += handler; try { webBrowser.Navigate(url); await tcs.Task; // wait for DocumentCompleted } finally { webBrowser.DocumentCompleted -= handler; } } // get the root element var documentElement = webBrowser.Document.GetElementsByTagName("html")[0]; // poll the current HTML for changes asynchronosly var html = documentElement.OuterHtml; while (true) { // wait asynchronously, this will throw if cancellation requested await Task.Delay(POLL_DELAY, token); // continue polling if the WebBrowser is still busy if (webBrowser.IsBusy) continue; var htmlNow = documentElement.OuterHtml; if (html == htmlNow) break; // no changes detected, end the poll loop html = htmlNow; } // consider the page fully rendered token.ThrowIfCancellationRequested(); return html; } // enable HTML5 (assuming we're running IE10+) // more info: https://stackoverflow.com/a/18333982/1768303 public static void SetFeatureBrowserEmulation() { if (System.ComponentModel.LicenseManager.UsageMode != System.ComponentModel.LicenseUsageMode.Runtime) return; var appName = System.IO.Path.GetFileName(System.Diagnostics.Process.GetCurrentProcess().MainModule.FileName); Registry.SetValue(@"HKEY_CURRENT_USER\Software\Microsoft\Internet Explorer\Main\FeatureControl\FEATURE_BROWSER_EMULATION", appName, 10000, RegistryValueKind.DWord); } } /// /// MessageLoopApartment /// STA thread with message pump for serial execution of tasks /// by Noseratio - https://stackoverflow.com/a/22262976/1768303 /// public class MessageLoopApartment : IDisposable { Thread _thread; // the STA thread TaskScheduler _taskScheduler; // the STA thread's task scheduler public TaskScheduler TaskScheduler { get { return _taskScheduler; } } /// MessageLoopApartment constructor public MessageLoopApartment() { var tcs = new TaskCompletionSource(); // start an STA thread and gets a task scheduler _thread = new Thread(startArg => { EventHandler idleHandler = null; idleHandler = (s, e) => { // handle Application.Idle just once Application.Idle -= idleHandler; // return the task scheduler tcs.SetResult(TaskScheduler.FromCurrentSynchronizationContext()); }; // handle Application.Idle just once // to make sure we're inside the message loop // and SynchronizationContext has been correctly installed Application.Idle += idleHandler; Application.Run(); }); _thread.SetApartmentState(ApartmentState.STA); _thread.IsBackground = true; _thread.Start(); _taskScheduler = tcs.Task.Result; } /// shutdown the STA thread public void Dispose() { if (_taskScheduler != null) { var taskScheduler = _taskScheduler; _taskScheduler = null; // execute Application.ExitThread() on the STA thread Task.Factory.StartNew( () => Application.ExitThread(), CancellationToken.None, TaskCreationOptions.None, taskScheduler).Wait(); _thread.Join(); _thread = null; } } /// Task.Factory.StartNew wrappers public void Invoke(Action action) { Task.Factory.StartNew(action, CancellationToken.None, TaskCreationOptions.None, _taskScheduler).Wait(); } public TResult Invoke(Func action) { return Task.Factory.StartNew(action, CancellationToken.None, TaskCreationOptions.None, _taskScheduler).Result; } public Task Run(Action action, CancellationToken token) { return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler); } public Task Run(Func action, CancellationToken token) { return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler); } public Task Run(Func action, CancellationToken token) { return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler).Unwrap(); } public Task Run(Func<Task> action, CancellationToken token) { return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler).Unwrap(); } }
}
一种解决方案是使用SemaphoreSlim来维护有限的WebBrowser对象池以并行地废弃网站。 为所有WebBrowser实例共享公共消息循环也是有意义的。
以下是基于我链接的控制台网络剪贴器代码实现的方法 。 新部分是WebBrowserPool类（警告：仅经过轻微测试）：
using Microsoft.Win32; using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Windows.Forms; namespace AsyncWebBrowserScrapper { class Program { // by Noseratio - https://stackoverflow.com/a/23819021/1768303 // test: web-scrap a list of URLs static async Task ScrapSitesAsync(string[] urls, CancellationToken token) { using (var pool = new WebBrowserPool(maxParallel: 2, token: token)) { // cancel each site in 30s or when the main token is signalled var timeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds; var results = urls.ToDictionary( url => url, url => pool.ScrapSiteAsync(url, timeout)); await Task.WhenAll(results.Values); foreach (var url in results.Keys) { Console.WriteLine("URL:\n" + url); string html = results[url].Result; Console.WriteLine("HTML:\n" + html); } } } // entry point static void Main(string[] args) { try { WebBrowserExt.SetFeatureBrowserEmulation(); // enable HTML5 var cts = new CancellationTokenSource((int)TimeSpan.FromMinutes(3).TotalMilliseconds); var task = ScrapSitesAsync( new[] { "http://example.com", "http://example.org", "http://example.net", "http://www.bing.com", "http://www.google.com" }, cts.Token); task.Wait(); Console.WriteLine("Press Enter to exit..."); Console.ReadLine(); } catch (Exception ex) { while (ex is AggregateException && ex.InnerException != null) ex = ex.InnerException; Console.WriteLine(ex.Message); Environment.Exit(-1); } } } /// /// WebBrowserPool the pool of WebBrowser objects sharing the same message loop /// public class WebBrowserPool : IDisposable { MessageLoopApartment _apartment; // a WinFroms STA thread with message loop readonly SemaphoreSlim _semaphore; // regulate available browsers readonly Queue _browsers; // the pool of available browsers readonly HashSet _pendingTasks; // keep track of pending tasks for proper cancellation readonly CancellationTokenSource _cts; // global cancellation (for Dispose) public WebBrowserPool(int maxParallel, CancellationToken token) { if (maxParallel < 1) throw new ArgumentException("maxParallel"); _cts = CancellationTokenSource.CreateLinkedTokenSource(token); _apartment = new MessageLoopApartment(); _semaphore = new SemaphoreSlim(maxParallel); _browsers = new Queue(); _pendingTasks = new HashSet(); // init the pool of WebBrowser objects _apartment.Invoke(() => { while (--maxParallel >= 0) _browsers.Enqueue(new WebBrowser()); }); } // Navigate to a site and get a snapshot of its DOM HTML public async Task ScrapSiteAsync(string url, int timeout, CancellationToken token = default(CancellationToken)) { var navigationCts = CancellationTokenSource.CreateLinkedTokenSource(token, _cts.Token); var combinedToken = navigationCts.Token; // we have a limited number of WebBrowser objects available, so await the semaphore await _semaphore.WaitAsync(combinedToken); try { if (timeout != Timeout.Infinite) navigationCts.CancelAfter(timeout); // run the main logic on the STA thread return await _apartment.Run(async () => { // acquire the 1st available WebBrowser from the pool var webBrowser = _browsers.Dequeue(); try { var task = webBrowser.NavigateAsync(url, combinedToken); _pendingTasks.Add(task); // register the pending task try { return await task; } finally { // unregister the completed task _pendingTasks.Remove(task); } } finally { // return the WebBrowser to the pool _browsers.Enqueue(webBrowser); } }, combinedToken); } finally { _semaphore.Release(); } } // Dispose of WebBrowserPool public void Dispose() { if (_apartment == null) throw new ObjectDisposedException(this.GetType().Name); // cancel and wait for all pending tasks _cts.Cancel(); var task = _apartment.Run(() => Task.WhenAll(_pendingTasks.ToArray())); try { task.Wait(); } catch { if (!task.IsCanceled) throw; } // dispose of WebBrowser objects _apartment.Run(() => { while (_browsers.Any()) _browsers.Dequeue().Dispose(); }); _apartment.Dispose(); _apartment = null; } } /// /// WebBrowserExt - WebBrowser extensions /// by Noseratio - https://stackoverflow.com/a/22262976/1768303 /// public static class WebBrowserExt { const int POLL_DELAY = 500; // navigate and download public static async Task NavigateAsync(this WebBrowser webBrowser, string url, CancellationToken token) { // navigate and await DocumentCompleted var tcs = new TaskCompletionSource(); WebBrowserDocumentCompletedEventHandler handler = (s, arg) => tcs.TrySetResult(true); using (token.Register( () => { webBrowser.Stop(); tcs.TrySetCanceled(); }, useSynchronizationContext: true)) { webBrowser.DocumentCompleted += handler; try { webBrowser.Navigate(url); await tcs.Task; // wait for DocumentCompleted } finally { webBrowser.DocumentCompleted -= handler; } } // get the root element var documentElement = webBrowser.Document.GetElementsByTagName("html")[0]; // poll the current HTML for changes asynchronosly var html = documentElement.OuterHtml; while (true) { // wait asynchronously, this will throw if cancellation requested await Task.Delay(POLL_DELAY, token); // continue polling if the WebBrowser is still busy if (webBrowser.IsBusy) continue; var htmlNow = documentElement.OuterHtml; if (html == htmlNow) break; // no changes detected, end the poll loop html = htmlNow; } // consider the page fully rendered token.ThrowIfCancellationRequested(); return html; } // enable HTML5 (assuming we're running IE10+) // more info: https://stackoverflow.com/a/18333982/1768303 public static void SetFeatureBrowserEmulation() { if (System.ComponentModel.LicenseManager.UsageMode != System.ComponentModel.LicenseUsageMode.Runtime) return; var appName = System.IO.Path.GetFileName(System.Diagnostics.Process.GetCurrentProcess().MainModule.FileName); Registry.SetValue(@"HKEY_CURRENT_USER\Software\Microsoft\Internet Explorer\Main\FeatureControl\FEATURE_BROWSER_EMULATION", appName, 10000, RegistryValueKind.DWord); } } /// /// MessageLoopApartment /// STA thread with message pump for serial execution of tasks /// by Noseratio - https://stackoverflow.com/a/22262976/1768303 /// public class MessageLoopApartment : IDisposable { Thread _thread; // the STA thread TaskScheduler _taskScheduler; // the STA thread's task scheduler public TaskScheduler TaskScheduler { get { return _taskScheduler; } } /// MessageLoopApartment constructor public MessageLoopApartment() { var tcs = new TaskCompletionSource(); // start an STA thread and gets a task scheduler _thread = new Thread(startArg => { EventHandler idleHandler = null; idleHandler = (s, e) => { // handle Application.Idle just once Application.Idle -= idleHandler; // return the task scheduler tcs.SetResult(TaskScheduler.FromCurrentSynchronizationContext()); }; // handle Application.Idle just once // to make sure we're inside the message loop // and SynchronizationContext has been correctly installed Application.Idle += idleHandler; Application.Run(); }); _thread.SetApartmentState(ApartmentState.STA); _thread.IsBackground = true; _thread.Start(); _taskScheduler = tcs.Task.Result; } /// shutdown the STA thread public void Dispose() { if (_taskScheduler != null) { var taskScheduler = _taskScheduler; _taskScheduler = null; // execute Application.ExitThread() on the STA thread Task.Factory.StartNew( () => Application.ExitThread(), CancellationToken.None, TaskCreationOptions.None, taskScheduler).Wait(); _thread.Join(); _thread = null; } } /// Task.Factory.StartNew wrappers public void Invoke(Action action) { Task.Factory.StartNew(action, CancellationToken.None, TaskCreationOptions.None, _taskScheduler).Wait(); } public TResult Invoke(Func action) { return Task.Factory.StartNew(action, CancellationToken.None, TaskCreationOptions.None, _taskScheduler).Result; } public Task Run(Action action, CancellationToken token = default(CancellationToken)) { return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler); } public Task Run (Func action, CancellationToken token = default(CancellationToken)) { return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler); } public Task Run(Func action, CancellationToken token = default(CancellationToken)) { return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler).Unwrap(); } public Task Run (Func> action, CancellationToken token = default(CancellationToken)) { return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler).Unwrap(); } } }