using System; using System.Collections.Generic; using System.Text; using System.Net.Sockets; using System.IO; using System.Threading; using CommonLang; using CommonNetwork; using CommonNetwork.Sockets; using CommonLang.IO; using CommonLang.Log; using System.Net; using CommonLang.Net; using System.Net.Security; using System.Security.Cryptography.X509Certificates; using System.Security.Authentication; namespace CommonNetwork.Http { public delegate void HttpConnectHandler(WebClient www); public delegate void HttpPostHandler(string result); public delegate void HttpGetHandler(byte[] result); public class HttpRequest { public const string METHOD_GET = "GET"; public const string METHOD_POST = "POST"; public const string CONTENT_TYPE_OCTET_STREAM = "application/octet-stream"; public const string CONTENT_TYPE_WWW_FORM_URLENCODED = "application/x-www-form-urlencoded"; public const string CONTENT_TYPE_TEXT_XML = "text/xml"; public string Method = METHOD_GET; public string ContentType = CONTENT_TYPE_OCTET_STREAM; public string Referer; public string Accept = "text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2"; public string AcceptLanguage = "zh-CN"; public string AcceptEncoding = "identity"; public string UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"; public string Connection = "keep-alive"; public string CacheControl = "no-cache"; public SslProtocols SslProtocol = SslProtocols.Ssl3; public X509CertificateCollection Certificates = null; public RemoteCertificateValidationCallback OnRemoteCertificateValidation = null; public LocalCertificateSelectionCallback OnLocalCertificateValidation = null; private Properties _params; public Properties Params { get { if (_params == null) { _params = new Properties(); } return _params; } } public byte[] Content; } public class HttpResponse { public Properties Params { get; internal set; } public string Status { get; internal set; } public string ContentType { get; internal set; } public int ContentLength { get; internal set; } public string Location { get; internal set; } public bool IsGzip { get; internal set; } public bool IsChunk { get; internal set; } private Stream input; public Stream InputStream { get { return input; } internal set { if (this.IsChunk) { this.input = new ChunkInputStream(this, value); } else { this.input = value; } } } public override string ToString() { var sb = new StringBuilder(); if (Params != null) { foreach (var e in Params) { sb.Append(e.Key + " : " + e.Value + WebClient.BR); } } return sb.ToString(); } public byte[] ReadContentToEnd() { if (IsChunk) { byte[] data = IOUtil.ReadToEnd(input); return data; } else { byte[] data = new byte[ContentLength]; IOUtil.ReadToEnd(input, data, 0, data.Length); return data; } } internal class ChunkInputStream : Stream { private readonly HttpResponse response; private readonly Stream baseStream; private int current_chunk_pos = 0; private int current_chunk_size = -1; public override bool CanRead { get { return baseStream.CanRead; } } public override bool CanSeek { get { return false; } } public override bool CanWrite { get { return false; } } public override long Length { get { return 0; } } public override long Position { get { return 0; } set { } } internal ChunkInputStream(HttpResponse rsp, Stream s) { this.response = rsp; this.baseStream = s; } public override int Read(byte[] buffer, int offset, int count) { if (current_chunk_pos >= current_chunk_size) { try { var line = WebClient.ReadLine(baseStream); current_chunk_size = Convert.ToInt32(line.Trim(), 16); current_chunk_pos = 0; } catch (Exception err) { throw new Exception("Maybe exists 'chunk-ext' field.", err); } } if (current_chunk_size == 0) { return 0; } int total = current_chunk_size - current_chunk_pos; count = Math.Min(total, count); int readed = baseStream.Read(buffer, offset, count); if (readed > 0) { current_chunk_pos += readed; if (current_chunk_pos >= current_chunk_size) { WebClient.ReadLine(baseStream); } } return readed; } public override long Seek(long offset, SeekOrigin origin) { throw new NotImplementedException(); } public override void SetLength(long value) { throw new NotImplementedException(); } public override void Flush() { throw new NotImplementedException(); } public override void Write(byte[] buffer, int offset, int count) { throw new NotImplementedException(); } } } public class WebClient : IDisposable { private static Logger _log; private static Logger log { get { if (_log == null) _log = LoggerFactory.GetLogger("WebClient"); return _log; } } private TcpClient mSocket = null; private Uri url; public HttpRequest Request = new HttpRequest(); public HttpResponse Response { get; private set; } public Exception Error { get; private set; } public int TimeoutMS { get; set; } public WebClient(Uri url) { this.url = url; this.TimeoutMS = 30000; } public void Dispose() { try { if (mSocket != null) { mSocket.Close(); } } catch (System.Exception e) { log.Warn(e.Message, e); } } public Stream Connect() { _connect(null); if (Response != null) { return Response.InputStream; } return null; } public void ConnectAsync(HttpConnectHandler handler) { var ts = new ThreadStart(() => { this._connect(handler); }); var tr = new Thread(ts); tr.IsBackground = true; tr.Start(); } //----------------------------------------------------------------------------------------------------------------- #region Internal /// /// /// /// 强制IPv6 /// /// private static TcpClient _connect_remote(bool forceIPv6, Uri location, int timeoutMS) { if (location.HostNameType == UriHostNameType.Dns) { Console.WriteLine("dns : " + location + " " + location.HostNameType.ToString()); var ips = Dns.GetHostEntry(location.Host); // 如果只包含IPv6地址,表示当前环境IPv6 only var family = IPUtil.IsOnlyIPv6(ips) ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork; var mSocket = new TcpClient(forceIPv6 ? AddressFamily.InterNetworkV6 : family); mSocket.SendTimeout = timeoutMS; mSocket.ReceiveTimeout = timeoutMS; if (family != AddressFamily.InterNetworkV6 && forceIPv6) { //首次是IPV6地址,优先选择V6地址// foreach (var ip in ips.AddressList) { if (ip.AddressFamily == AddressFamily.InterNetworkV6) { mSocket.Connect(ip, location.Port); return mSocket; } } //强转V4地址到V6地址// foreach (var ip in ips.AddressList) { if (ip.AddressFamily == AddressFamily.InterNetwork) { var ipv6 = IPUtil.MapToIPv6(ip); Console.WriteLine("ipv4 to ipv6 : " + ip + " - " + ipv6); mSocket.Connect(ipv6, location.Port); return mSocket; } } mSocket.Connect(ips.AddressList, location.Port); } else { mSocket.Connect(ips.AddressList, location.Port); } return mSocket; } else { Console.WriteLine("ip : " + location + " " + location.HostNameType.ToString()); var mSocket = new TcpClient(forceIPv6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork); mSocket.SendTimeout = timeoutMS; mSocket.ReceiveTimeout = timeoutMS; if (location.HostNameType != UriHostNameType.IPv6 && forceIPv6) { var ipv6 = IPUtil.MapToIPv6(location.Host); Console.WriteLine("ipv4 to ipv6 : " + location.Host + " - " + ipv6); mSocket.Connect(ipv6, location.Port); } else { mSocket.Connect(location.Host, location.Port); } return mSocket; } } private void _connect(HttpConnectHandler handler) { try { bool isIpV6 = false; Uri location = url; do { mSocket = _connect_remote(isIpV6, location, TimeoutMS); if (!mSocket.Connected) { return; } log.Debug("RemoteEndPoint AddressFamily : " + mSocket.Client.RemoteEndPoint.AddressFamily); if (mSocket.Client.RemoteEndPoint.AddressFamily == AddressFamily.InterNetworkV6) { isIpV6 = true; } Stream stream = null; if (location.Scheme == "http") { stream = mSocket.GetStream(); } else if (location.Scheme == "https") { var sslStream = new SslStream(mSocket.GetStream(), true, Request.OnRemoteCertificateValidation, Request.OnLocalCertificateValidation); sslStream.ReadTimeout = TimeoutMS; sslStream.WriteTimeout = TimeoutMS; var certificates = Request.Certificates; if (certificates != null) { var store = new X509Store(StoreName.My); certificates = store.Certificates; } sslStream.AuthenticateAsClient(location.Host, certificates, Request.SslProtocol, false); if (!sslStream.IsAuthenticated) { return; } stream = sslStream; } else { log.Error("url must start with HTTP or HTTPS:" + url); break; } log.Info("send request : " + location); this.Response = _send_request(stream, location, Request.Referer, Request); log.Info("response : " + Response.Status); // redirect 302 if (!string.IsNullOrEmpty(this.Response.Location)) { if (mSocket != null) { try { stream.Close(); } catch (System.Exception e) { log.Warn(e.Message, e); } try { mSocket.Close(); } catch (System.Exception e) { log.Warn(e.Message, e); } } log.Info(" redirect to : " + this.Response.Location); location = new Uri(this.Response.Location); continue; } else { this.Response.InputStream = stream; break; } } while (true); } catch (System.Exception e) { log.Error(e.Message, e); this.Response = null; this.Error = e; } finally { if (handler != null) { handler(this); } } } private static HttpResponse _send_request(Stream stream, Uri url, string referer, HttpRequest request) { // Send if (request.Method.Equals(HttpRequest.METHOD_GET)) { StringBuilder header_text = new StringBuilder(); header_text.Append(request.Method + " " + url.PathAndQuery + " HTTP/1.1" + BR); header_text.Append("Accept: " + request.Accept + BR); header_text.Append("Referer: " + referer + BR); header_text.Append("Accept-Language: " + request.AcceptLanguage + BR); header_text.Append("Accept-Encoding: " + request.AcceptEncoding + BR); header_text.Append("User-Agent: " + request.UserAgent + BR); header_text.Append("Host: " + url.Host + BR); header_text.Append("Connection: " + request.Connection + BR); header_text.Append("Cache-Control: " + request.CacheControl + BR); foreach (KeyValuePair e in request.Params) { header_text.Append(e.Key + ": " + e.Value + BR); } header_text.Append(BR); // send HTTP GET // string req = header_text.ToString(); byte[] header_bytes = Encoding.UTF8.GetBytes(req); IOUtil.WriteToEnd(stream, header_bytes, 0, header_bytes.Length); } else if (request.Method.Equals(HttpRequest.METHOD_POST)) { byte[] body = _get_post_data(url, request); // send HTTP Post Head // StringBuilder header_text = new StringBuilder(); header_text.Append(request.Method + " " + url.AbsolutePath + " HTTP/1.1" + BR); header_text.Append("Host: " + url.Host + BR); header_text.Append("Referer: " + referer + BR); header_text.Append("Content-Length: " + body.Length + BR); header_text.Append("Content-Type: " + request.ContentType + BR); header_text.Append("User-Agent: " + request.UserAgent + BR); header_text.Append("Connection: " + request.Connection + BR); header_text.Append("Cache-Control: " + request.CacheControl + BR); header_text.Append("Accept: " + request.Accept + BR); foreach (KeyValuePair e in request.Params) { header_text.Append(e.Key + ": " + e.Value + BR); } header_text.Append(BR); string req = header_text.ToString(); byte[] header_bytes = Encoding.UTF8.GetBytes(req); IOUtil.WriteToEnd(stream, header_bytes, 0, header_bytes.Length); IOUtil.WriteToEnd(stream, body, 0, body.Length); } HttpResponse response = new HttpResponse(); response.Params = new Properties(); String line = null; while ((line = ReadLine(stream)) != null) { if (line.Length == 0) { break; } else if (line.ToUpper().StartsWith("HTTP")) { response.Status = line; } else if (response.Params.ParseLine(line, ":")) { } string len = null; if (TryGetResponseValue(line, "Content-Length", out len)) { response.ContentLength = int.Parse(len); } else if (TryGetResponseValue(line, "Content-Type", out len)) { response.ContentType = len; } else if (TryGetResponseValue(line, "Location", out len)) { response.Location = len; } else if (TryGetResponseValue(line, "Content-Encoding", out len)) { if (len.IndexOf("gzip", StringComparison.OrdinalIgnoreCase) >= 0) { response.IsGzip = true; } } else if (TryGetResponseValue(line, "Transfer-Encoding", out len)) { if (len.IndexOf("chunked", StringComparison.OrdinalIgnoreCase) >= 0) { response.IsChunk = true; } } } return response; } private static byte[] _get_post_data(Uri url, HttpRequest request) { if (request.Content == null) { string body = url.Query; if (body.StartsWith("?")) { body = body.Substring(1); } int body_length = Encoding.UTF8.GetByteCount(body); byte[] body_bytes = Encoding.UTF8.GetBytes(body); return body_bytes; } else { return request.Content; } } #endregion //----------------------------------------------------------------------------------------------------------------- #region STATIC public static string DownloadString(Uri url, Encoding enc = null) { if (enc == null) { enc = CUtils.UTF8; } using (WebClient client = new WebClient(url)) { client.Request.Method = HttpRequest.METHOD_GET; client.Request.Referer = url.AbsoluteUri; client.Connect(); byte[] data = client.Response.ReadContentToEnd(); string ret = enc.GetString(data); return ret; } } public static byte[] Get(Uri url) { using (WebClient client = new WebClient(url)) { client.Request.Method = HttpRequest.METHOD_GET; client.Request.Referer = url.AbsoluteUri; client.Connect(); byte[] data = client.Response.ReadContentToEnd(); return data; } } public static void GetAsync(Uri url, HttpGetHandler handler) { WebClient client = new WebClient(url); client.Request.Method = HttpRequest.METHOD_GET; client.Request.Referer = url.AbsoluteUri; client.ConnectAsync((www) => { try { if (www.Response != null) { byte[] data = client.Response.ReadContentToEnd(); handler.Invoke(data); } else { handler.Invoke(null); } } catch (Exception err) { log.Error(err.Message, err); handler.Invoke(null); } finally { www.Dispose(); } }); } public static string Post(Uri url, string referer = null, Encoding enc = null) { if (enc == null) { enc = CUtils.UTF8; } if (referer == null) { referer = url.AbsoluteUri; } using (WebClient client = new WebClient(url)) { client.Request.Method = HttpRequest.METHOD_POST; client.Request.ContentType = "application/x-www-form-urlencoded"; client.Request.Referer = referer; client.Connect(); byte[] data = client.Response.ReadContentToEnd(); string ret = enc.GetString(data); return ret; } } public static void PostAsync(Uri url, string referer, Encoding enc, HttpPostHandler handler) { if (enc == null) { enc = CUtils.UTF8; } if (referer == null) { referer = url.AbsoluteUri; } WebClient client = new WebClient(url); client.Request.Method = HttpRequest.METHOD_POST; client.Request.ContentType = "application/x-www-form-urlencoded"; client.Request.Referer = referer; client.ConnectAsync((www) => { try { if (www.Response != null) { byte[] data = client.Response.ReadContentToEnd(); string ret = enc.GetString(data); handler.Invoke(ret); } else { handler.Invoke(null); } } catch (Exception err) { log.Error(err.Message, err); handler.Invoke(null); } finally { www.Dispose(); } }); } public static void PostAsync(Uri url, Encoding enc, HttpPostHandler handler) { PostAsync(url, null, CUtils.UTF8, handler); } public static void PostAsync(Uri url, HttpPostHandler handler) { PostAsync(url, CUtils.UTF8, handler); } #endregion //---------------------------------------------------------------------------------------------- #region UTILS public static readonly string BR = "\r\n"; public static readonly char[] SPLIT = new char[] { ':' }; public static string FormatPath(string path) { path = path.Replace('\\', '/'); return path; } private static bool TryGetResponseValue(string line, string key, out string value) { if (line.ToLower().StartsWith(key.ToLower())) { string[] kv = line.Split(SPLIT, 2); value = kv[1].Trim(); return true; } value = null; return false; } public static string ReadLine(Stream input) { byte _r = (byte)'\r'; byte _n = (byte)'\n'; using (var ms = MemoryStreamObjectPool.AllocAutoRelease()) { int a0 = 0; while (a0 >= 0) { int a1 = input.ReadByte(); if (a1 < 0) { break; } ms.WriteByte((byte)a1); if (a0 == _r && a1 == _n) { break; } a0 = a1; } ms.Flush(); return Encoding.ASCII.GetString(ms.GetBuffer(), 0, (int)(ms.Length - 2)); } } #endregion } }