使用C#编写网络爬虫时，并发控制和IP池管理是保障爬取效率与稳定性的核心环节，不合理的并发设置容易导致目标站点反爬封禁，而IP池管理不当则会造成资源浪费或爬取中断。
一、C#网络爬虫的并发控制实现
并发控制的核心是平衡爬取速度和目标站点的访问压力，避免短时间内发送过多请求触发反爬机制。C#中常用的并发控制方案有以下几种：
1. 基于SemaphoreSlim的信号量控制
SemaphoreSlim是轻量级的信号量实现，适合控制同时执行的请求数量，示例代码如下：
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
class Crawler
{
// 最大并发数，可根据目标站点承受能力调整
private static readonly int MaxConcurrentRequests = 5;
private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
private static readonly HttpClient _httpClient = new HttpClient();
static async Task Main(string[] args)
{
List<string> urls = new List<string>
{
"http://ipipp.com/page1",
"http://ipipp.com/page2",
"http://ipipp.com/page3",
"http://ipipp.com/page4",
"http://ipipp.com/page5",
"http://ipipp.com/page6"
};
List<Task> tasks = new List<Task>();
foreach (var url in urls)
{
tasks.Add(ProcessUrlAsync(url));
}
await Task.WhenAll(tasks);
Console.WriteLine("所有请求处理完成");
}
static async Task ProcessUrlAsync(string url)
{
// 等待信号量，控制并发数量
await _semaphore.WaitAsync();
try
{
Console.WriteLine($"开始请求：{url}");
HttpResponseMessage response = await _httpClient.GetAsync(url);
response.EnsureSuccessStatusCode();
string content = await response.Content.ReadAsStringAsync();
Console.WriteLine($"请求{url}完成，内容长度：{content.Length}");
}
catch (Exception ex)
{
Console.WriteLine($"请求{url}失败：{ex.Message}");
}
finally
{
// 释放信号量，允许其他请求执行
_semaphore.Release();
}
}
}
2. 基于线程池的并发控制
可以通过设置线程池的最大工作线程数来控制并发，适合CPU密集型爬取场景，示例代码如下：
using System;
using System.Net.Http;
using System.Threading;
class ThreadPoolCrawler
{
static void Main(string[] args)
{
// 设置线程池最大工作线程数为5
ThreadPool.SetMaxThreads(5, 5);
string[] urls = {
"http://ipipp.com/page1",
"http://ipipp.com/page2",
"http://ipipp.com/page3"
};
foreach (var url in urls)
{
ThreadPool.QueueUserWorkItem(ProcessUrl, url);
}
// 等待所有任务完成
Thread.Sleep(5000);
Console.WriteLine("爬取完成");
}
static void ProcessUrl(object state)
{
string url = state as string;
try
{
using (HttpClient client = new HttpClient())
{
string content = client.GetStringAsync(url).Result;
Console.WriteLine($"请求{url}完成，长度：{content.Length}");
}
}
catch (Exception ex)
{
Console.WriteLine($"请求{url}失败：{ex.Message}");
}
}
}
二、C#网络爬虫的IP池管理实现
IP池管理的核心是维护可用代理IP列表，实现IP的校验、调度、失效替换，避免单一IP被封禁。
1. IP池的基础结构定义
首先定义代理IP的实体类和IP池的基础结构：
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
// 代理IP实体类
public class ProxyIp
{
// IP地址
public string Ip { get; set; }
// 端口
public int Port { get; set; }
// 最后校验时间
public DateTime LastCheckTime { get; set; }
// 是否可用
public bool IsAvailable { get; set; }
}
// IP池管理类
public class IpPoolManager
{
// 存储可用代理IP的并发集合
private readonly ConcurrentBag<ProxyIp> _availableProxies = new ConcurrentBag<ProxyIp>();
// 存储待校验的代理IP
private readonly Queue<ProxyIp> _pendingProxies = new Queue<ProxyIp>();
// 校验目标地址，用于测试代理是否可用
private readonly string _checkUrl = "http://ipipp.com/checkip";
// 校验超时时间
private readonly int _checkTimeout = 3000;
// 添加代理IP到待校验队列
public void AddProxy(string ip, int port)
{
_pendingProxies.Enqueue(new ProxyIp
{
Ip = ip,
Port = port,
LastCheckTime = DateTime.Now,
IsAvailable = false
});
}
// 校验单个代理IP是否可用
public async Task<bool> CheckProxyAsync(ProxyIp proxy)
{
try
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy($"{proxy.Ip}:{proxy.Port}"),
UseProxy = true
};
using (var client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromMilliseconds(_checkTimeout);
var response = await client.GetAsync(_checkUrl);
proxy.LastCheckTime = DateTime.Now;
proxy.IsAvailable = response.IsSuccessStatusCode;
return proxy.IsAvailable;
}
}
catch
{
proxy.LastCheckTime = DateTime.Now;
proxy.IsAvailable = false;
return false;
}
}
}
2. IP池的调度与更新逻辑
IP池需要定期校验失效IP，补充新IP，同时实现请求时的IP调度，示例代码如下：
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
public class IpPoolManager
{
private readonly ConcurrentBag<ProxyIp> _availableProxies = new ConcurrentBag<ProxyIp>();
private readonly Queue<ProxyIp> _pendingProxies = new Queue<ProxyIp>();
private readonly string _checkUrl = "http://ipipp.com/checkip";
private readonly int _checkTimeout = 3000;
// 定时校验间隔，单位毫秒
private readonly int _checkInterval = 60000;
private readonly Timer _checkTimer;
private readonly Random _random = new Random();
public IpPoolManager()
{
// 初始化定时校验任务
_checkTimer = new Timer(CheckPoolAsync, null, _checkInterval, _checkInterval);
}
// 添加代理IP到待校验队列
public void AddProxy(string ip, int port)
{
_pendingProxies.Enqueue(new ProxyIp
{
Ip = ip,
Port = port,
LastCheckTime = DateTime.Now,
IsAvailable = false
});
}
// 定时校验IP池
private async void CheckPoolAsync(object state)
{
// 校验待校验队列中的IP
while (_pendingProxies.Count > 0)
{
var proxy = _pendingProxies.Dequeue();
if (await CheckProxyAsync(proxy))
{
_availableProxies.Add(proxy);
}
}
// 校验已可用IP是否仍然有效
var availableList = _availableProxies.ToList();
_availableProxies.Clear();
foreach (var proxy in availableList)
{
if (DateTime.Now - proxy.LastCheckTime > TimeSpan.FromMinutes(5))
{
if (await CheckProxyAsync(proxy))
{
_availableProxies.Add(proxy);
}
}
else
{
_availableProxies.Add(proxy);
}
}
Console.WriteLine($"IP池校验完成，当前可用IP数量：{_availableProxies.Count}");
}
// 校验单个代理IP
public async Task<bool> CheckProxyAsync(ProxyIp proxy)
{
try
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy($"{proxy.Ip}:{proxy.Port}"),
UseProxy = true
};
using (var client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromMilliseconds(_checkTimeout);
var response = await client.GetAsync(_checkUrl);
proxy.LastCheckTime = DateTime.Now;
proxy.IsAvailable = response.IsSuccessStatusCode;
return proxy.IsAvailable;
}
}
catch
{
proxy.LastCheckTime = DateTime.Now;
proxy.IsAvailable = false;
return false;
}
}
// 获取一个可用代理IP
public ProxyIp GetProxy()
{
if (_availableProxies.TryTake(out var proxy))
{
return proxy;
}
return null;
}
// 使用代理IP发送请求
public async Task<string> RequestWithProxyAsync(string url)
{
var proxy = GetProxy();
if (proxy == null)
{
throw new Exception("当前无可用代理IP");
}
try
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy($"{proxy.Ip}:{proxy.Port}"),
UseProxy = true
};
using (var client = new HttpClient(handler))
{
var response = await client.GetAsync(url);
response.EnsureSuccessStatusCode();
return await response.Content.ReadAsStringAsync();
}
}
catch (Exception ex)
{
// 请求失败则标记IP不可用，重新放入待校验队列
proxy.IsAvailable = false;
_pendingProxies.Enqueue(proxy);
throw new Exception($"使用代理{proxy.Ip}:{proxy.Port}请求失败：{ex.Message}");
}
}
}
三、并发控制与IP池的协同使用
将并发控制和IP池结合，才能实现稳定高效的爬取，协同使用的示例代码如下：
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
class CombinedCrawler
{
private static readonly int MaxConcurrentRequests = 3;
private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
private static readonly IpPoolManager _ipPoolManager = new IpPoolManager();
static async Task Main(string[] args)
{
// 初始化IP池，添加测试代理IP，实际使用时可对接代理IP接口获取
_ipPoolManager.AddProxy("127.0.0.1", 8080);
_ipPoolManager.AddProxy("192.168.0.1", 8888);
List<string> urls = new List<string>
{
"http://ipipp.com/data1",
"http://ipipp.com/data2",
"http://ipipp.com/data3",
"http://ipipp.com/data4"
};
List<Task> tasks = new List<Task>();
foreach (var url in urls)
{
tasks.Add(CrawlUrlAsync(url));
}
await Task.WhenAll(tasks);
Console.WriteLine("所有爬取任务完成");
}
static async Task CrawlUrlAsync(string url)
{
await _semaphore.WaitAsync();
try
{
Console.WriteLine($"开始爬取：{url}");
string content = await _ipPoolManager.RequestWithProxyAsync(url);
Console.WriteLine($"爬取{url}完成，内容长度：{content.Length}");
}
catch (Exception ex)
{
Console.WriteLine($"爬取{url}失败：{ex.Message}");
}
finally
{
_semaphore.Release();
}
}
}
四、注意事项
- 并发数量需要根据目标站点的反爬规则调整，建议从低到高逐步测试，避免触发封禁。
- IP池中的代理IP需要定期校验，失效IP及时移除，同时补充新的可用IP。
- 请求时需要设置合理的超时时间，避免长时间等待无响应请求占用资源。
- 爬取过程中建议添加请求间隔，进一步降低被反爬的概率。