using System;
using System.Collections.Generic;
using System.Text;
using System.Net.Sockets;
using System.IO;
using System.Threading;
using CommonLang;
using CommonNetwork;
using CommonNetwork.Sockets;
using CommonLang.IO;
using CommonLang.Log;
using System.Net;
using CommonLang.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Security.Authentication;
namespace CommonNetwork.Http
{
public delegate void HttpConnectHandler(WebClient www);
public delegate void HttpPostHandler(string result);
public delegate void HttpGetHandler(byte[] result);
public class HttpRequest
{
public const string METHOD_GET = "GET";
public const string METHOD_POST = "POST";
public const string CONTENT_TYPE_OCTET_STREAM = "application/octet-stream";
public const string CONTENT_TYPE_WWW_FORM_URLENCODED = "application/x-www-form-urlencoded";
public const string CONTENT_TYPE_TEXT_XML = "text/xml";
public string Method = METHOD_GET;
public string ContentType = CONTENT_TYPE_OCTET_STREAM;
public string Referer;
public string Accept = "text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2";
public string AcceptLanguage = "zh-CN";
public string AcceptEncoding = "identity";
public string UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36";
public string Connection = "keep-alive";
public string CacheControl = "no-cache";
public SslProtocols SslProtocol = SslProtocols.Ssl3;
public X509CertificateCollection Certificates = null;
public RemoteCertificateValidationCallback OnRemoteCertificateValidation = null;
public LocalCertificateSelectionCallback OnLocalCertificateValidation = null;
private Properties _params;
public Properties Params
{
get
{
if (_params == null)
{
_params = new Properties();
}
return _params;
}
}
public byte[] Content;
}
public class HttpResponse
{
public Properties Params { get; internal set; }
public string Status { get; internal set; }
public string ContentType { get; internal set; }
public int ContentLength { get; internal set; }
public string Location { get; internal set; }
public bool IsGzip { get; internal set; }
public bool IsChunk { get; internal set; }
private Stream input;
public Stream InputStream
{
get { return input; }
internal set
{
if (this.IsChunk)
{
this.input = new ChunkInputStream(this, value);
}
else
{
this.input = value;
}
}
}
public override string ToString()
{
var sb = new StringBuilder();
if (Params != null)
{
foreach (var e in Params)
{
sb.Append(e.Key + " : " + e.Value + WebClient.BR);
}
}
return sb.ToString();
}
public byte[] ReadContentToEnd()
{
if (IsChunk)
{
byte[] data = IOUtil.ReadToEnd(input);
return data;
}
else
{
byte[] data = new byte[ContentLength];
IOUtil.ReadToEnd(input, data, 0, data.Length);
return data;
}
}
internal class ChunkInputStream : Stream
{
private readonly HttpResponse response;
private readonly Stream baseStream;
private int current_chunk_pos = 0;
private int current_chunk_size = -1;
public override bool CanRead { get { return baseStream.CanRead; } }
public override bool CanSeek { get { return false; } }
public override bool CanWrite { get { return false; } }
public override long Length { get { return 0; } }
public override long Position { get { return 0; } set { } }
internal ChunkInputStream(HttpResponse rsp, Stream s)
{
this.response = rsp;
this.baseStream = s;
}
public override int Read(byte[] buffer, int offset, int count)
{
if (current_chunk_pos >= current_chunk_size)
{
try
{
var line = WebClient.ReadLine(baseStream);
current_chunk_size = Convert.ToInt32(line.Trim(), 16);
current_chunk_pos = 0;
}
catch (Exception err)
{
throw new Exception("Maybe exists 'chunk-ext' field.", err);
}
}
if (current_chunk_size == 0)
{
return 0;
}
int total = current_chunk_size - current_chunk_pos;
count = Math.Min(total, count);
int readed = baseStream.Read(buffer, offset, count);
if (readed > 0)
{
current_chunk_pos += readed;
if (current_chunk_pos >= current_chunk_size)
{
WebClient.ReadLine(baseStream);
}
}
return readed;
}
public override long Seek(long offset, SeekOrigin origin)
{
throw new NotImplementedException();
}
public override void SetLength(long value)
{
throw new NotImplementedException();
}
public override void Flush()
{
throw new NotImplementedException();
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
}
}
}
public class WebClient : IDisposable
{
private static Logger _log;
private static Logger log
{
get
{
if (_log == null) _log = LoggerFactory.GetLogger("WebClient");
return _log;
}
}
private TcpClient mSocket = null;
private Uri url;
public HttpRequest Request = new HttpRequest();
public HttpResponse Response { get; private set; }
public Exception Error { get; private set; }
public int TimeoutMS { get; set; }
public WebClient(Uri url)
{
this.url = url;
this.TimeoutMS = 30000;
}
public void Dispose()
{
try
{
if (mSocket != null)
{
mSocket.Close();
}
}
catch (System.Exception e)
{
log.Warn(e.Message, e);
}
}
public Stream Connect()
{
_connect(null);
if (Response != null)
{
return Response.InputStream;
}
return null;
}
public void ConnectAsync(HttpConnectHandler handler)
{
var ts = new ThreadStart(() => { this._connect(handler); });
var tr = new Thread(ts);
tr.IsBackground = true;
tr.Start();
}
//-----------------------------------------------------------------------------------------------------------------
#region Internal
///
///
///
/// 强制IPv6
///
///
private static TcpClient _connect_remote(bool forceIPv6, Uri location, int timeoutMS)
{
if (location.HostNameType == UriHostNameType.Dns)
{
Console.WriteLine("dns : " + location + " " + location.HostNameType.ToString());
var ips = Dns.GetHostEntry(location.Host);
// 如果只包含IPv6地址,表示当前环境IPv6 only
var family = IPUtil.IsOnlyIPv6(ips) ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
var mSocket = new TcpClient(forceIPv6 ? AddressFamily.InterNetworkV6 : family);
mSocket.SendTimeout = timeoutMS;
mSocket.ReceiveTimeout = timeoutMS;
if (family != AddressFamily.InterNetworkV6 && forceIPv6)
{
//首次是IPV6地址,优先选择V6地址//
foreach (var ip in ips.AddressList)
{
if (ip.AddressFamily == AddressFamily.InterNetworkV6)
{
mSocket.Connect(ip, location.Port);
return mSocket;
}
}
//强转V4地址到V6地址//
foreach (var ip in ips.AddressList)
{
if (ip.AddressFamily == AddressFamily.InterNetwork)
{
var ipv6 = IPUtil.MapToIPv6(ip);
Console.WriteLine("ipv4 to ipv6 : " + ip + " - " + ipv6);
mSocket.Connect(ipv6, location.Port);
return mSocket;
}
}
mSocket.Connect(ips.AddressList, location.Port);
}
else
{
mSocket.Connect(ips.AddressList, location.Port);
}
return mSocket;
}
else
{
Console.WriteLine("ip : " + location + " " + location.HostNameType.ToString());
var mSocket = new TcpClient(forceIPv6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork);
mSocket.SendTimeout = timeoutMS;
mSocket.ReceiveTimeout = timeoutMS;
if (location.HostNameType != UriHostNameType.IPv6 && forceIPv6)
{
var ipv6 = IPUtil.MapToIPv6(location.Host);
Console.WriteLine("ipv4 to ipv6 : " + location.Host + " - " + ipv6);
mSocket.Connect(ipv6, location.Port);
}
else
{
mSocket.Connect(location.Host, location.Port);
}
return mSocket;
}
}
private void _connect(HttpConnectHandler handler)
{
try
{
bool isIpV6 = false;
Uri location = url;
do
{
mSocket = _connect_remote(isIpV6, location, TimeoutMS);
if (!mSocket.Connected)
{
return;
}
log.Debug("RemoteEndPoint AddressFamily : " + mSocket.Client.RemoteEndPoint.AddressFamily);
if (mSocket.Client.RemoteEndPoint.AddressFamily == AddressFamily.InterNetworkV6)
{
isIpV6 = true;
}
Stream stream = null;
if (location.Scheme == "http")
{
stream = mSocket.GetStream();
}
else if (location.Scheme == "https")
{
var sslStream = new SslStream(mSocket.GetStream(), true, Request.OnRemoteCertificateValidation, Request.OnLocalCertificateValidation);
sslStream.ReadTimeout = TimeoutMS;
sslStream.WriteTimeout = TimeoutMS;
var certificates = Request.Certificates;
if (certificates != null)
{
var store = new X509Store(StoreName.My);
certificates = store.Certificates;
}
sslStream.AuthenticateAsClient(location.Host, certificates, Request.SslProtocol, false);
if (!sslStream.IsAuthenticated)
{
return;
}
stream = sslStream;
}
else
{
log.Error("url must start with HTTP or HTTPS:" + url);
break;
}
log.Info("send request : " + location);
this.Response = _send_request(stream, location, Request.Referer, Request);
log.Info("response : " + Response.Status);
// redirect 302
if (!string.IsNullOrEmpty(this.Response.Location))
{
if (mSocket != null)
{
try { stream.Close(); } catch (System.Exception e) { log.Warn(e.Message, e); }
try { mSocket.Close(); } catch (System.Exception e) { log.Warn(e.Message, e); }
}
log.Info(" redirect to : " + this.Response.Location);
location = new Uri(this.Response.Location);
continue;
}
else
{
this.Response.InputStream = stream;
break;
}
}
while (true);
}
catch (System.Exception e)
{
log.Error(e.Message, e);
this.Response = null;
this.Error = e;
}
finally
{
if (handler != null)
{
handler(this);
}
}
}
private static HttpResponse _send_request(Stream stream, Uri url, string referer, HttpRequest request)
{
// Send
if (request.Method.Equals(HttpRequest.METHOD_GET))
{
StringBuilder header_text = new StringBuilder();
header_text.Append(request.Method + " " + url.PathAndQuery + " HTTP/1.1" + BR);
header_text.Append("Accept: " + request.Accept + BR);
header_text.Append("Referer: " + referer + BR);
header_text.Append("Accept-Language: " + request.AcceptLanguage + BR);
header_text.Append("Accept-Encoding: " + request.AcceptEncoding + BR);
header_text.Append("User-Agent: " + request.UserAgent + BR);
header_text.Append("Host: " + url.Host + BR);
header_text.Append("Connection: " + request.Connection + BR);
header_text.Append("Cache-Control: " + request.CacheControl + BR);
foreach (KeyValuePair e in request.Params)
{
header_text.Append(e.Key + ": " + e.Value + BR);
}
header_text.Append(BR);
// send HTTP GET //
string req = header_text.ToString();
byte[] header_bytes = Encoding.UTF8.GetBytes(req);
IOUtil.WriteToEnd(stream, header_bytes, 0, header_bytes.Length);
}
else if (request.Method.Equals(HttpRequest.METHOD_POST))
{
byte[] body = _get_post_data(url, request);
// send HTTP Post Head //
StringBuilder header_text = new StringBuilder();
header_text.Append(request.Method + " " + url.AbsolutePath + " HTTP/1.1" + BR);
header_text.Append("Host: " + url.Host + BR);
header_text.Append("Referer: " + referer + BR);
header_text.Append("Content-Length: " + body.Length + BR);
header_text.Append("Content-Type: " + request.ContentType + BR);
header_text.Append("User-Agent: " + request.UserAgent + BR);
header_text.Append("Connection: " + request.Connection + BR);
header_text.Append("Cache-Control: " + request.CacheControl + BR);
header_text.Append("Accept: " + request.Accept + BR);
foreach (KeyValuePair e in request.Params)
{
header_text.Append(e.Key + ": " + e.Value + BR);
}
header_text.Append(BR);
string req = header_text.ToString();
byte[] header_bytes = Encoding.UTF8.GetBytes(req);
IOUtil.WriteToEnd(stream, header_bytes, 0, header_bytes.Length);
IOUtil.WriteToEnd(stream, body, 0, body.Length);
}
HttpResponse response = new HttpResponse();
response.Params = new Properties();
String line = null;
while ((line = ReadLine(stream)) != null)
{
if (line.Length == 0)
{
break;
}
else if (line.ToUpper().StartsWith("HTTP"))
{
response.Status = line;
}
else if (response.Params.ParseLine(line, ":"))
{
}
string len = null;
if (TryGetResponseValue(line, "Content-Length", out len))
{
response.ContentLength = int.Parse(len);
}
else if (TryGetResponseValue(line, "Content-Type", out len))
{
response.ContentType = len;
}
else if (TryGetResponseValue(line, "Location", out len))
{
response.Location = len;
}
else if (TryGetResponseValue(line, "Content-Encoding", out len))
{
if (len.IndexOf("gzip", StringComparison.OrdinalIgnoreCase) >= 0)
{
response.IsGzip = true;
}
}
else if (TryGetResponseValue(line, "Transfer-Encoding", out len))
{
if (len.IndexOf("chunked", StringComparison.OrdinalIgnoreCase) >= 0)
{
response.IsChunk = true;
}
}
}
return response;
}
private static byte[] _get_post_data(Uri url, HttpRequest request)
{
if (request.Content == null)
{
string body = url.Query;
if (body.StartsWith("?"))
{
body = body.Substring(1);
}
int body_length = Encoding.UTF8.GetByteCount(body);
byte[] body_bytes = Encoding.UTF8.GetBytes(body);
return body_bytes;
}
else
{
return request.Content;
}
}
#endregion
//-----------------------------------------------------------------------------------------------------------------
#region STATIC
public static string DownloadString(Uri url, Encoding enc = null)
{
if (enc == null)
{
enc = CUtils.UTF8;
}
using (WebClient client = new WebClient(url))
{
client.Request.Method = HttpRequest.METHOD_GET;
client.Request.Referer = url.AbsoluteUri;
client.Connect();
byte[] data = client.Response.ReadContentToEnd();
string ret = enc.GetString(data);
return ret;
}
}
public static byte[] Get(Uri url)
{
using (WebClient client = new WebClient(url))
{
client.Request.Method = HttpRequest.METHOD_GET;
client.Request.Referer = url.AbsoluteUri;
client.Connect();
byte[] data = client.Response.ReadContentToEnd();
return data;
}
}
public static void GetAsync(Uri url, HttpGetHandler handler)
{
WebClient client = new WebClient(url);
client.Request.Method = HttpRequest.METHOD_GET;
client.Request.Referer = url.AbsoluteUri;
client.ConnectAsync((www) =>
{
try
{
if (www.Response != null)
{
byte[] data = client.Response.ReadContentToEnd();
handler.Invoke(data);
}
else
{
handler.Invoke(null);
}
}
catch (Exception err)
{
log.Error(err.Message, err);
handler.Invoke(null);
}
finally
{
www.Dispose();
}
});
}
public static string Post(Uri url, string referer = null, Encoding enc = null)
{
if (enc == null)
{
enc = CUtils.UTF8;
}
if (referer == null)
{
referer = url.AbsoluteUri;
}
using (WebClient client = new WebClient(url))
{
client.Request.Method = HttpRequest.METHOD_POST;
client.Request.ContentType = "application/x-www-form-urlencoded";
client.Request.Referer = referer;
client.Connect();
byte[] data = client.Response.ReadContentToEnd();
string ret = enc.GetString(data);
return ret;
}
}
public static void PostAsync(Uri url, string referer, Encoding enc, HttpPostHandler handler)
{
if (enc == null)
{
enc = CUtils.UTF8;
}
if (referer == null)
{
referer = url.AbsoluteUri;
}
WebClient client = new WebClient(url);
client.Request.Method = HttpRequest.METHOD_POST;
client.Request.ContentType = "application/x-www-form-urlencoded";
client.Request.Referer = referer;
client.ConnectAsync((www) =>
{
try
{
if (www.Response != null)
{
byte[] data = client.Response.ReadContentToEnd();
string ret = enc.GetString(data);
handler.Invoke(ret);
}
else
{
handler.Invoke(null);
}
}
catch (Exception err)
{
log.Error(err.Message, err);
handler.Invoke(null);
}
finally
{
www.Dispose();
}
});
}
public static void PostAsync(Uri url, Encoding enc, HttpPostHandler handler)
{
PostAsync(url, null, CUtils.UTF8, handler);
}
public static void PostAsync(Uri url, HttpPostHandler handler)
{
PostAsync(url, CUtils.UTF8, handler);
}
#endregion
//----------------------------------------------------------------------------------------------
#region UTILS
public static readonly string BR = "\r\n";
public static readonly char[] SPLIT = new char[] { ':' };
public static string FormatPath(string path)
{
path = path.Replace('\\', '/');
return path;
}
private static bool TryGetResponseValue(string line, string key, out string value)
{
if (line.ToLower().StartsWith(key.ToLower()))
{
string[] kv = line.Split(SPLIT, 2);
value = kv[1].Trim();
return true;
}
value = null;
return false;
}
public static string ReadLine(Stream input)
{
byte _r = (byte)'\r';
byte _n = (byte)'\n';
using (var ms = MemoryStreamObjectPool.AllocAutoRelease())
{
int a0 = 0;
while (a0 >= 0)
{
int a1 = input.ReadByte();
if (a1 < 0)
{
break;
}
ms.WriteByte((byte)a1);
if (a0 == _r && a1 == _n)
{
break;
}
a0 = a1;
}
ms.Flush();
return Encoding.ASCII.GetString(ms.GetBuffer(), 0, (int)(ms.Length - 2));
}
}
#endregion
}
}