WSHCommon/Common/CommonNetwork/Http/WebClient.cs

using System;
using System.Collections.Generic;
using System.Text;
using System.Net.Sockets;
using System.IO;
using System.Threading;
using CommonLang;
using CommonNetwork;
using CommonNetwork.Sockets;
using CommonLang.IO;
using CommonLang.Log;
using System.Net;
using CommonLang.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Security.Authentication;

namespace CommonNetwork.Http
{
    public delegate void HttpConnectHandler(WebClient www);
    public delegate void HttpPostHandler(string result);
    public delegate void HttpGetHandler(byte[] result);

    public class HttpRequest
    {
        public const string METHOD_GET = "GET";
        public const string METHOD_POST = "POST";

        public const string CONTENT_TYPE_OCTET_STREAM = "application/octet-stream";
        public const string CONTENT_TYPE_WWW_FORM_URLENCODED = "application/x-www-form-urlencoded";
        public const string CONTENT_TYPE_TEXT_XML = "text/xml";

        public string Method = METHOD_GET;
        public string ContentType = CONTENT_TYPE_OCTET_STREAM;
        public string Referer;
        public string Accept = "text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2";
        public string AcceptLanguage = "zh-CN";
        public string AcceptEncoding = "identity";
        public string UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36";
        public string Connection = "keep-alive";
        public string CacheControl = "no-cache";

        public SslProtocols SslProtocol = SslProtocols.Ssl3;
        public X509CertificateCollection Certificates = null;
        public RemoteCertificateValidationCallback OnRemoteCertificateValidation = null;
        public LocalCertificateSelectionCallback OnLocalCertificateValidation = null;

        private Properties _params;
        public Properties Params
        {
            get
            {
                if (_params == null)
                {
                    _params = new Properties();
                }
                return _params;
            }
        }

        public byte[] Content;

    }

    public class HttpResponse
    {
        public Properties Params { get; internal set; }
        public string Status { get; internal set; }
        public string ContentType { get; internal set; }
        public int ContentLength { get; internal set; }
        public string Location { get; internal set; }
        public bool IsGzip { get; internal set; }
        public bool IsChunk { get; internal set; }

        private Stream input;
        public Stream InputStream
        {
            get { return input; }
            internal set
            {
                if (this.IsChunk)
                {
                    this.input = new ChunkInputStream(this, value);
                }
                else
                {
                    this.input = value;
                }
            }
        }

        public override string ToString()
        {
            var sb = new StringBuilder();
            if (Params != null)
            {
                foreach (var e in Params)
                {
                    sb.Append(e.Key + " : " + e.Value + WebClient.BR);
                }
            }
            return sb.ToString();
        }

        public byte[] ReadContentToEnd()
        {
            if (IsChunk)
            {
                byte[] data = IOUtil.ReadToEnd(input);
                return data;
            }
            else
            {
                byte[] data = new byte[ContentLength];
                IOUtil.ReadToEnd(input, data, 0, data.Length);
                return data;
            }
        }

        internal class ChunkInputStream : Stream
        {
            private readonly HttpResponse response;
            private readonly Stream baseStream;
            private int current_chunk_pos = 0;
            private int current_chunk_size = -1;

            public override bool CanRead { get { return baseStream.CanRead; } }
            public override bool CanSeek { get { return false; } }
            public override bool CanWrite { get { return false; } }
            public override long Length { get { return 0; } }
            public override long Position { get { return 0; } set { } }


            internal ChunkInputStream(HttpResponse rsp, Stream s)
            {
                this.response = rsp;
                this.baseStream = s;
            }
            public override int Read(byte[] buffer, int offset, int count)
            {
                if (current_chunk_pos >= current_chunk_size)
                {
                    try
                    {
                        var line = WebClient.ReadLine(baseStream);
                        current_chunk_size = Convert.ToInt32(line.Trim(), 16);
                        current_chunk_pos = 0;
                    }
                    catch (Exception err)
                    {
                        throw new Exception("Maybe exists 'chunk-ext' field.", err);
                    }
                }
                if (current_chunk_size == 0)
                {
                    return 0;
                }
                int total = current_chunk_size - current_chunk_pos;
                count = Math.Min(total, count);
                int readed = baseStream.Read(buffer, offset, count);
                if (readed > 0)
                {
                    current_chunk_pos += readed;
                    if (current_chunk_pos >= current_chunk_size)
                    {
                        WebClient.ReadLine(baseStream);
                    }
                }
                return readed;
            }
            public override long Seek(long offset, SeekOrigin origin)
            {
                throw new NotImplementedException();
            }
            public override void SetLength(long value)
            {
                throw new NotImplementedException();
            }
            public override void Flush()
            {
                throw new NotImplementedException();
            }
            public override void Write(byte[] buffer, int offset, int count)
            {
                throw new NotImplementedException();
            }
        }
    }

    public class WebClient : IDisposable
    {
        private static Logger _log;
        private static Logger log
        {
            get
            {
                if (_log == null) _log = LoggerFactory.GetLogger("WebClient");
                return _log;
            }
        }

        private TcpClient mSocket = null;
        private Uri url;

        public HttpRequest Request = new HttpRequest();
        public HttpResponse Response { get; private set; }
        public Exception Error { get; private set; }

        public int TimeoutMS { get; set; }

        public WebClient(Uri url)
        {
            this.url = url;
            this.TimeoutMS = 30000;
        }
        public void Dispose()
        {
            try
            {
                if (mSocket != null)
                {
                    mSocket.Close();
                }
            }
            catch (System.Exception e) 
            { 
                log.Warn(e.Message, e); 
            }
        }
        public Stream Connect()
        {
            _connect(null);
            if (Response != null)
            {
                return Response.InputStream;
            }
            return null;
        }
        public void ConnectAsync(HttpConnectHandler handler)
        {
            var ts = new ThreadStart(() => { this._connect(handler); });
            var tr = new Thread(ts);
            tr.IsBackground = true;
            tr.Start();
        }

        //-----------------------------------------------------------------------------------------------------------------

        #region Internal

        /// <summary>
        /// 
        /// </summary>
        /// <param name="forceIPv6">强制IPv6</param>
        /// <param name="location"></param>
        /// <param name="timeoutMS"></param>
        private static TcpClient _connect_remote(bool forceIPv6, Uri location, int timeoutMS)
        {
            if (location.HostNameType == UriHostNameType.Dns)
            {
                Console.WriteLine("dns : " + location + "  " + location.HostNameType.ToString());
                var ips = Dns.GetHostEntry(location.Host);
                // 如果只包含IPv6地址，表示当前环境IPv6 only
                var family = IPUtil.IsOnlyIPv6(ips) ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
                var mSocket = new TcpClient(forceIPv6 ? AddressFamily.InterNetworkV6 : family);
                mSocket.SendTimeout = timeoutMS;
                mSocket.ReceiveTimeout = timeoutMS;
                if (family != AddressFamily.InterNetworkV6 && forceIPv6)
                {
                    //首次是IPV6地址，优先选择V6地址//
                    foreach (var ip in ips.AddressList)
                    {
                        if (ip.AddressFamily == AddressFamily.InterNetworkV6)
                        {
                            mSocket.Connect(ip, location.Port);
                            return mSocket;
                        }
                    }
                    //强转V4地址到V6地址//
                    foreach (var ip in ips.AddressList)
                    {
                        if (ip.AddressFamily == AddressFamily.InterNetwork)
                        {
                            var ipv6 = IPUtil.MapToIPv6(ip);
                            Console.WriteLine("ipv4 to ipv6 : " + ip + " - " + ipv6);
                            mSocket.Connect(ipv6, location.Port);
                            return mSocket;
                        }
                    }
                    mSocket.Connect(ips.AddressList, location.Port);
                }
                else
                {
                    mSocket.Connect(ips.AddressList, location.Port);
                }
                return mSocket;
            }
            else
            {
                Console.WriteLine("ip : " + location + "  " + location.HostNameType.ToString());
                var mSocket = new TcpClient(forceIPv6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork);
                mSocket.SendTimeout = timeoutMS;
                mSocket.ReceiveTimeout = timeoutMS;
                if (location.HostNameType != UriHostNameType.IPv6 && forceIPv6)
                {
                    var ipv6 = IPUtil.MapToIPv6(location.Host);
                    Console.WriteLine("ipv4 to ipv6 : " + location.Host + " - " + ipv6);
                    mSocket.Connect(ipv6, location.Port);
                }
                else
                {
                    mSocket.Connect(location.Host, location.Port);
                }
                return mSocket;
            }
        }

        private void _connect(HttpConnectHandler handler)
        {
            try
            {
                bool isIpV6 = false;
                Uri location = url;
                do
                {
                    mSocket = _connect_remote(isIpV6, location, TimeoutMS);
                    if (!mSocket.Connected)
                    {
                        return;
                    }
                    log.Debug("RemoteEndPoint AddressFamily : " + mSocket.Client.RemoteEndPoint.AddressFamily);
                    if (mSocket.Client.RemoteEndPoint.AddressFamily == AddressFamily.InterNetworkV6)
                    {
                        isIpV6 = true;
                    }
                    Stream stream = null;
                    if (location.Scheme == "http")
                    {
                        stream = mSocket.GetStream();
                    }
                    else if (location.Scheme == "https")
                    {
                        var sslStream = new SslStream(mSocket.GetStream(), true, Request.OnRemoteCertificateValidation, Request.OnLocalCertificateValidation);
                        sslStream.ReadTimeout = TimeoutMS;
                        sslStream.WriteTimeout = TimeoutMS;
                        var certificates = Request.Certificates;
                        if (certificates != null)
                        {
                            var store = new X509Store(StoreName.My);
                            certificates = store.Certificates;
                        }
                        sslStream.AuthenticateAsClient(location.Host, certificates, Request.SslProtocol, false);
                        if (!sslStream.IsAuthenticated)
                        {
                            return;
                        }
                        stream = sslStream;
                    }
                    else
                    {
                        log.Error("url must start with HTTP or HTTPS:" + url);
                        break;
                    }
                    log.Info("send request : " + location);
                    this.Response = _send_request(stream, location, Request.Referer, Request);
                    log.Info("response : " + Response.Status);
                    // redirect 302
                    if (!string.IsNullOrEmpty(this.Response.Location))
                    {
                        if (mSocket != null)
                        {
                            try { stream.Close(); } catch (System.Exception e) { log.Warn(e.Message, e); }
                            try { mSocket.Close(); } catch (System.Exception e) { log.Warn(e.Message, e); }
                        }
                        log.Info(" redirect to : " + this.Response.Location);
                        location = new Uri(this.Response.Location);
                        continue;
                    }
                    else
                    {
                        this.Response.InputStream = stream;
                        break;
                    }
                }
                while (true);
            }
            catch (System.Exception e)
            {
                log.Error(e.Message, e);
                this.Response = null;
                this.Error = e;
            }
            finally
            {
                if (handler != null)
                {
                    handler(this);
                }
            }
        }

        private static HttpResponse _send_request(Stream stream, Uri url, string referer, HttpRequest request)
        {
            // Send
            if (request.Method.Equals(HttpRequest.METHOD_GET))
            {
                StringBuilder header_text = new StringBuilder();
                header_text.Append(request.Method + " " + url.PathAndQuery + " HTTP/1.1" + BR);
                header_text.Append("Accept: " + request.Accept + BR);
                header_text.Append("Referer: " + referer + BR);
                header_text.Append("Accept-Language: " + request.AcceptLanguage + BR);
                header_text.Append("Accept-Encoding: " + request.AcceptEncoding + BR);
                header_text.Append("User-Agent: " + request.UserAgent + BR);
                header_text.Append("Host: " + url.Host + BR);
                header_text.Append("Connection: " + request.Connection + BR);
                header_text.Append("Cache-Control: " + request.CacheControl + BR);
                foreach (KeyValuePair<string, string> e in request.Params)
                {
                    header_text.Append(e.Key + ": " + e.Value + BR);
                }
                header_text.Append(BR);
                // send HTTP GET //
                string req = header_text.ToString();
                byte[] header_bytes = Encoding.UTF8.GetBytes(req);
                IOUtil.WriteToEnd(stream, header_bytes, 0, header_bytes.Length);
            }
            else if (request.Method.Equals(HttpRequest.METHOD_POST))
            {
                byte[] body = _get_post_data(url, request);
                // send HTTP Post Head //
                StringBuilder header_text = new StringBuilder();
                header_text.Append(request.Method + " " + url.AbsolutePath + " HTTP/1.1" + BR);
                header_text.Append("Host: " + url.Host + BR);
                header_text.Append("Referer: " + referer + BR);
                header_text.Append("Content-Length: " + body.Length + BR);
                header_text.Append("Content-Type: " + request.ContentType + BR);
                header_text.Append("User-Agent: " + request.UserAgent + BR);
                header_text.Append("Connection: " + request.Connection + BR);
                header_text.Append("Cache-Control: " + request.CacheControl + BR);
                header_text.Append("Accept: " + request.Accept + BR);
                foreach (KeyValuePair<string, string> e in request.Params)
                {
                    header_text.Append(e.Key + ": " + e.Value + BR);
                }
                header_text.Append(BR);
                string req = header_text.ToString();
                byte[] header_bytes = Encoding.UTF8.GetBytes(req);
                IOUtil.WriteToEnd(stream, header_bytes, 0, header_bytes.Length);
                IOUtil.WriteToEnd(stream, body, 0, body.Length);
            }

            HttpResponse response = new HttpResponse();
            response.Params = new Properties();
            String line = null;
            while ((line = ReadLine(stream)) != null)
            {
                if (line.Length == 0)
                {
                    break;
                }
                else if (line.ToUpper().StartsWith("HTTP"))
                {
                    response.Status = line;
                }
                else if (response.Params.ParseLine(line, ":"))
                {

                }
                string len = null;
                if (TryGetResponseValue(line, "Content-Length", out len))
                {
                    response.ContentLength = int.Parse(len);
                }
                else if (TryGetResponseValue(line, "Content-Type", out len))
                {
                    response.ContentType = len;
                }
                else if (TryGetResponseValue(line, "Location", out len))
                {
                    response.Location = len;
                }
                else if (TryGetResponseValue(line, "Content-Encoding", out len))
                {
                    if (len.IndexOf("gzip", StringComparison.OrdinalIgnoreCase) >= 0)
                    {
                        response.IsGzip = true;
                    }
                }
                else if (TryGetResponseValue(line, "Transfer-Encoding", out len))
                {
                    if (len.IndexOf("chunked", StringComparison.OrdinalIgnoreCase) >= 0)
                    {
                        response.IsChunk = true;
                    }
                }
            }
            return response;
        }

        private static byte[] _get_post_data(Uri url, HttpRequest request)
        {
            if (request.Content == null)
            {
                string body = url.Query;
                if (body.StartsWith("?"))
                {
                    body = body.Substring(1);
                }
                int body_length = Encoding.UTF8.GetByteCount(body);
                byte[] body_bytes = Encoding.UTF8.GetBytes(body);
                return body_bytes;
            }
            else
            {
                return request.Content;
            }
        }


        #endregion

        //-----------------------------------------------------------------------------------------------------------------

        #region STATIC

        public static string DownloadString(Uri url, Encoding enc = null)
        {
            if (enc == null)
            {
                enc = CUtils.UTF8;
            }
            using (WebClient client = new WebClient(url))
            {
                client.Request.Method = HttpRequest.METHOD_GET;
                client.Request.Referer = url.AbsoluteUri;
                client.Connect();
                byte[] data = client.Response.ReadContentToEnd();
                string ret = enc.GetString(data);
                return ret;
            }
        }

        public static byte[] Get(Uri url)
        {
            using (WebClient client = new WebClient(url))
            {
                client.Request.Method = HttpRequest.METHOD_GET;
                client.Request.Referer = url.AbsoluteUri;
                client.Connect();
                byte[] data = client.Response.ReadContentToEnd();
                return data;
            }
        }

        public static void GetAsync(Uri url, HttpGetHandler handler)
        {
            WebClient client = new WebClient(url);
            client.Request.Method = HttpRequest.METHOD_GET;
            client.Request.Referer = url.AbsoluteUri;
            client.ConnectAsync((www) =>
            {
                try
                {
                    if (www.Response != null)
                    {
                        byte[] data = client.Response.ReadContentToEnd();
                        handler.Invoke(data);
                    }
                    else
                    {
                        handler.Invoke(null);
                    }
                }
                catch (Exception err)
                {
                    log.Error(err.Message, err);
                    handler.Invoke(null);
                }
                finally
                {
                    www.Dispose();
                }
            });
        }

        public static string Post(Uri url, string referer = null, Encoding enc = null)
        {
            if (enc == null)
            {
                enc = CUtils.UTF8;
            }
            if (referer == null)
            {
                referer = url.AbsoluteUri;
            }
            using (WebClient client = new WebClient(url))
            {
                client.Request.Method = HttpRequest.METHOD_POST;
                client.Request.ContentType = "application/x-www-form-urlencoded";
                client.Request.Referer = referer;
                client.Connect();
                byte[] data = client.Response.ReadContentToEnd();
                string ret = enc.GetString(data);
                return ret;
            }
        }
        public static void PostAsync(Uri url, string referer, Encoding enc, HttpPostHandler handler)
        {
            if (enc == null)
            {
                enc = CUtils.UTF8;
            }
            if (referer == null)
            {
                referer = url.AbsoluteUri;
            }
            WebClient client = new WebClient(url);
            client.Request.Method = HttpRequest.METHOD_POST;
            client.Request.ContentType = "application/x-www-form-urlencoded";
            client.Request.Referer = referer;
            client.ConnectAsync((www) =>
            {
                try
                {
                    if (www.Response != null)
                    {
                        byte[] data = client.Response.ReadContentToEnd();
                        string ret = enc.GetString(data);
                        handler.Invoke(ret);
                    }
                    else
                    {
                        handler.Invoke(null);
                    }
                }
                catch (Exception err)
                {
                    log.Error(err.Message, err);
                    handler.Invoke(null);
                }
                finally
                {
                    www.Dispose();
                }
            });
        }
        public static void PostAsync(Uri url, Encoding enc, HttpPostHandler handler)
        {
            PostAsync(url, null, CUtils.UTF8, handler);
        }
        public static void PostAsync(Uri url, HttpPostHandler handler)
        {
            PostAsync(url, CUtils.UTF8, handler);
        }

        #endregion

        //----------------------------------------------------------------------------------------------

        #region UTILS

        public static readonly string BR = "\r\n";
        public static readonly char[] SPLIT = new char[] { ':' };

        public static string FormatPath(string path)
        {
            path = path.Replace('\\', '/');
            return path;
        }

        private static bool TryGetResponseValue(string line, string key, out string value)
        {
            if (line.ToLower().StartsWith(key.ToLower()))
            {
                string[] kv = line.Split(SPLIT, 2);
                value = kv[1].Trim();
                return true;
            }
            value = null;
            return false;
        }

        public static string ReadLine(Stream input)
        {
            byte _r = (byte)'\r';
            byte _n = (byte)'\n';
            using (var ms = MemoryStreamObjectPool.AllocAutoRelease())
            {
                int a0 = 0;
                while (a0 >= 0)
                {
                    int a1 = input.ReadByte();
                    if (a1 < 0)
                    {
                        break;
                    }
                    ms.WriteByte((byte)a1);
                    if (a0 == _r && a1 == _n)
                    {
                        break;
                    }
                    a0 = a1;
                }
                ms.Flush();
                return Encoding.ASCII.GetString(ms.GetBuffer(), 0, (int)(ms.Length - 2));
            }
        }

        #endregion



    }


}