WSHCommon/Common/CommonFileSystem/Updater/Updater.cs

using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Threading;
using CommonLang;
using CommonLang.File;
using CommonLang.Concurrent;
using CommonLang.Log;

namespace MPQ.Updater
{
    public class MPQUpdater : IDisposable
    {
        private CommonLang.Log.Logger log = LoggerFactory.GetLogger("MPQUpdater");
        public static string ZIP_EXT = ".z";
        public static string MPQ_EXT = ".mpq";
        public static string VERSION_TEXT_BEGIN = "BEGIN";
        public static string VERSION_TEXT_END = "END";

        private static UTF8Encoding UTF8 = new UTF8Encoding(false);
        private static readonly char Separator = Path.DirectorySeparatorChar;

        private readonly MPQDirver driver;

        private Uri version_url;
        private FileInfo version_file;
        private string[] url_roots;

        private string version_text = "";

        private MPQUpdaterListener listener;
        private SyncMessageQueue<MPQUpdaterEvent> events = new SyncMessageQueue<MPQUpdaterEvent>();
        private bool is_running = false;
        private bool is_exit = false;

        private long last_update_time;
        private long last_update_download_bytes = 0;
        private long last_update_unzip_bytes = 0;
        private long current_download_speed_BPS = 0;
        private long current_unzip_spped_BPS = 0;

        private DirectoryInfo stream_root;
        private DirectoryInfo save_root;
        private bool is_check_md5 = false;
        private Thread workthread;
        private long total_unzip_bytes = 0;
        private long total_download_bytes = 0;
        private AtomicLong current_unzip_bytes = new AtomicLong(0);
        private AtomicLong current_download_bytes = new AtomicLong(0);
        private List<RemoteFileInfo> remoteFiles = new List<RemoteFileInfo>();
        private AtomicReference<Status> status = new AtomicReference<Status>(Status.NA);

        private long _mSaveAvaliable = 0L;

        public DirectoryInfo LocalSaveRoot { get { return save_root; } }

        public MPQUpdater(MPQDirver d = null)
        {
            if (d == null) { d = new MPQDirver(); }
            this.driver = d;
        }

        /// <summary>
        /// 创建自动更新程序
        /// </summary>
        /// <param name="remote_version_url">远程列表文件地址(http://localhost/xxxx/version.txt)</param>
        /// <param name="remote_version_prefix">远程下载地址根目录（多个备选）</param>
        /// <param name="version_suffix">下载资源类型后缀</param>
        /// <param name="local_save_root">本地存储目录</param>
        /// <param name="local_bundle_root">本地包内资源目录</param>
        /// <param name="validate_md5">是否验证MD5</param>
        /// <param name="listener">监听器</param>
        public void Init(
            Uri remote_version_url,
            string[] remote_version_prefix,
            string version_suffix,
            DirectoryInfo local_save_root,
            DirectoryInfo local_bundle_root,
            string mpq_txt,
            bool validate_md5,
            MPQUpdaterListener listener)
        {
            if (remote_version_prefix.Length == 0)
            {
                throw new Exception("remote_version_prefix length is 0 !!!");
            }
            if (local_save_root.Equals(local_bundle_root))
            {
                throw new Exception("save root cannot be bundle root !!!");
            }
            stream_root = local_bundle_root;
            this.save_root = local_save_root;
            this.version_file = new FileInfo(Path.GetFullPath(local_save_root.FullName + Separator + version_suffix));
            this.url_roots = remote_version_prefix;
            this.version_url = remote_version_url;
            this.is_check_md5 = validate_md5;
            versionText = mpq_txt;
            this.listener = listener;
            this.DownloadTimeoutSEC = 15;
        }

		public void Init(
			string[] remote_version_prefix,
			string version_suffix,
			DirectoryInfo local_save_root,
			DirectoryInfo local_bundle_root,
			bool validate_md5,
			MPQUpdaterListener listener)
		{
			if (remote_version_prefix.Length == 0)
			{
				throw new Exception("remote_version_prefix length is 0 !!!");
			}
			if (local_save_root.Equals(local_bundle_root))
			{
				throw new Exception("save root cannot be bundle root !!!");
			}
			this.save_root = local_save_root;// new DirectoryInfo(Path.GetFullPath(local_save_root));
			this.stream_root = local_bundle_root;// new DirectoryInfo(Path.GetFullPath(local_bundle_root));
			this.version_file = new FileInfo(Path.GetFullPath(local_save_root.FullName + Separator + version_suffix));
			this.url_roots = remote_version_prefix;
			this.version_url = new Uri(url_roots[0] + "/" + version_suffix);
			this.is_check_md5 = validate_md5;
			this.listener = listener;
			this.DownloadTimeoutSEC = 30;
		}

		Action<float> checkMpqUpdateCb = null;
        public void CheckNeedUpdate(bool userMpq, Action<float> checkCb)
        {
            if(userMpq)
            {
                checkMpqUpdateCb = checkCb;
                var thread = new Thread(CheckThread);
                thread.IsBackground = true;
                status.Value = Status.Checking;
                thread.Start();
            }
            else
            {
                IsLoadFinised = true;
                status.Value = Status.Done;
                checkCb(0);
            }
        }

        public float CheckProgress = 0f;
        public string versionText;
        public long needSize = 0;

        public Queue<RemoteFileInfo> need_files = new Queue<RemoteFileInfo>();
        void CheckThread()
        {
            remoteFiles.Clear();
            HashMap<string, RemoteFileInfo> _zip_files = new HashMap<string, RemoteFileInfo>();
            HashMap<string, RemoteFileInfo> _bin_files = new HashMap<string, RemoteFileInfo>();
            if(string.IsNullOrEmpty(versionText))
            {
                versionText = CommonNetwork.Http.WebClient.DownloadString(version_url, UTF8);
            }

            if (versionText.StartsWith(VERSION_TEXT_BEGIN) && versionText.EndsWith(VERSION_TEXT_END))
            {
                char[] spc = { ':' };
                string[] lines = versionText.Split('\n');
                int cur = 0;
                int len = lines.Length;
                foreach (string line in lines)
                {
                    string[] kv = line.Split(spc, 3);
                    if (kv.Length == 3)
                    {
                        string key = kv[2].Trim().Replace('\\', Separator);
                        string md5 = kv[0].Trim();
                        uint fsize = uint.Parse(kv[1].Trim());

                        //如果peristerpath有此文件
                        var _savePath = Path.GetFullPath(save_root.FullName + Separator + key);
                        FileInfo localFile = new FileInfo(_savePath);
                        RemoteFileInfo localinf = new RemoteFileInfo(md5, fsize, key, localFile);

                        remoteFiles.Add(localinf);

                        if (localFile.Exists)
                        {
                            if (localFile.Length == fsize)
                            {
                                if(key.EndsWith(ZIP_EXT))
                                {
                                    // 如果是没有解压的zip ,就直接放在need_files 进入下一步
                                    need_files.Enqueue(localinf);
                                }
                                cur++;
                                CheckProgress = cur * 1f / len;
                                continue;
                            }
                            else if (localFile.Length > fsize)
                            {
                                localFile.Delete();
                            }
                        }

                        if (key.ToLower().EndsWith(ZIP_EXT))
                        {
                            _zip_files[localFile.Name] = localinf;
                            needSize += localinf.size;
                        }
                        else if (!key.ToLower().EndsWith(MPQ_EXT))
                        {
                            _bin_files[localFile.Name] = localinf;
                            needSize += localinf.size;
                        }
                        else
                        {
                            cur++;
                            CheckProgress = cur * 1f / len;
                        }
                    }
                }

                List<RemoteFileInfo> trydownlist = new List<RemoteFileInfo>();
                trydownlist.AddRange(_zip_files.Values);
                trydownlist.AddRange(_bin_files.Values);
                foreach (RemoteFileInfo inf in trydownlist)
                {
                    // 确认对应的MPQ文件是否完整 //
                    if (inf.key.EndsWith(ZIP_EXT))
                    {
                        string mpq_name = inf.key.TrimEnd('.', 'z');
                        var mpq_file = remoteFiles.Find(data => data.key == mpq_name);
                        if (mpq_file != null && mpq_file.IsCompletion())
                        {
                            cur++;
                            CheckProgress = cur * 1f / len;
                            needSize -= inf.size;

                            inf.file.Delete();
                            continue;
                        }
                    }

                    // 如果已经存在未下载完成的 
                    if (inf.file.Exists)
                    {
                        needSize -= inf.size;
                        needSize += inf.NeedLoadSize();
                    }
                    need_files.Enqueue(inf);

                    cur++;
                    CheckProgress = cur * 1f / len;
                }
            }

            var mbytes = (float)Math.Round(needSize / (float)1048576, 2);
            if (checkMpqUpdateCb != null)
            {
                if (mbytes == 0)
                {
                    if(need_files.Count == 0)
                    {
                        IsLoadFinised = true;
                        status.Value = Status.Done;
                    }
                    else
                    {
                        IsLoadFinised = true;
                        status.Value = Status.Unzipping;
                    }
                }

                checkMpqUpdateCb(mbytes);
            }
        }

        /**
         * 开始自动更新 
         */
        public void Start()
        {
            _mSaveAvaliable = this.driver.GetAvaliableSpace(this.save_root.FullName);
            this.workthread = new Thread(new ThreadStart(Run));
            this.workthread.Name = "MPQUpdater";
            this.workthread.Start();
        }

        public bool HaveEnoughSpace(float size)
        {
            var uSize = (long)Math.Ceiling(size * 1048576);
            var space = driver.GetAvaliableSpace(save_root.FullName);
            return space > uSize;
        }

        public void Update()
        {
            long now_time = Environment.TickCount;
            if (Math.Abs(now_time - last_update_time) >= 1000)
            {
                double delta_time = now_time - last_update_time;
                double delta_download = current_download_bytes.Value - last_update_download_bytes;
                double delta_unzip = current_unzip_bytes.Value - last_update_unzip_bytes;

                this.current_download_speed_BPS = (long)(delta_download / delta_time * 1000.0);
                this.current_unzip_spped_BPS = (long)(delta_unzip / delta_time * 1000.0);

                this.last_update_time = now_time;
                this.last_update_download_bytes = current_download_bytes.Value;
                this.last_update_unzip_bytes = current_unzip_bytes.Value;
            }
            events.ProcessMessages(DoEvent);
        }

        private void QueueEvent(MPQUpdaterEvent evt)
        {
            listener.onEvent(this, evt);
        }

        public string GetThreadSattus()
        {
            log.Log(workthread.IsAlive.ToString());
            return workthread.ThreadState.ToString();
        }

        public void RunTry()
        {
            Start();
        }

        private void DoEvent(MPQUpdaterEvent e)
        {
            listener.onEvent(this, e);
        }

        public void Run()
        {
            try
            {
                new RunTask(this).Run();
            }
            catch (Exception err)
            {
                log.Error(err.Message, err);
            }
        }

        //----------------------------------------------------------------------------------------------------------
        public enum Status
        {
            Error = -1,
            NA = 0,
            Checking = 1,
            Downloading = 2,
            Unzipping = 3,
            Done = 4,
        }

        class RunTask
        {
            private CommonLang.Log.Logger log;
            private readonly MPQUpdater updater;

            private Queue<RemoteFileInfo> _zip_files = new Queue<RemoteFileInfo>();

            public RunTask(MPQUpdater updater)
            {
                this.log = updater.log;
                this.updater = updater;
            }

            public void Run()
            {
                updater.is_running = true;
                try
                {
                    // 先删除 资源服上 没有的资源
                    List<FileInfo> exists = CFiles.listAllFiles(updater.save_root);
                    foreach (FileInfo ff in exists)
                    {
                        var remoteAsset = updater.remoteFiles.Find(data => data.file.Name == ff.Name);
                        if (remoteAsset == null)
                        {
                            ff.Delete();
                        }
                    }

                    if (!updater.IsLoadFinised)
                    {
                        // 下载资源
                        updater.status.Value = Status.Downloading;
                    }

                    if (run_download_zips())
                    {
                        return;
                    }

                    updater.IsLoadFinised = true;

                    // 解压资源
                    updater.status.Value = Status.Unzipping;
                    if (run_unzip())
                    {
                        return;
                    }

                    updater.status.Value = Status.Done;
                }
                catch (Exception err)
                {
                    log.DebugFormat(err.Message, err);
                    updater.QueueEvent(new MPQUpdaterEvent(MPQUpdaterEvent.TYPE_ERROR, "ERROR : " + err.Message, err));
                }
                finally
                {
                    updater.is_running = false;
                }
            }

            private bool run_download_zips()
            {
                var needFiles = updater.need_files;

                while(needFiles.Count > 0)
                {
                    if (updater.is_exit) return true;  
                    var inf = needFiles.Dequeue();
                    try
                    {
                        long need_bytes = inf.size;
                        long exist_size = 0;
                        // 如果已经存在未下载完成的 //
                        if (inf.file.Exists)
                        {
                            exist_size = inf.file.Length;
                            need_bytes = inf.size - exist_size;
                            if (need_bytes == 0)
                            {
                                if (inf.key.EndsWith(ZIP_EXT))
                                {
                                    var str = inf.key.TrimEnd('.', 'z');
                                    var file = updater.remoteFiles.Find(data => data.key == str);
                                    if (file != null)
                                    {
                                        updater.total_unzip_bytes += file.size;
                                        _zip_files.Enqueue(inf);
                                    }
                                }
                                continue;
                            }
                        }
                        else
                        {
                            CFiles.createFile(inf.file);
                        }
                        // 如果需要的空间无法满足 //
//                        long save_avaliable = updater.driver.GetAvaliableSpace(updater.save_root.FullName);
                        long save_avaliable = updater._mSaveAvaliable;
                        if (need_bytes >= save_avaliable)
                        {
                            needFiles.Enqueue(inf);

                            updater.QueueEvent(new MPQUpdaterEvent(
                                MPQUpdaterEvent.TYPE_NOT_ENOUGH_SPACE, "Space not available!" ));
                            return true;
                        }

                        try
                        {
                            log.Info(string.Format("开始下载: {0} ", inf.key));

                            if (run_download_single(inf, exist_size, need_bytes))
                            {
                                needFiles.Enqueue(inf);
                            }
                            else
                            {
                                if (inf.key.EndsWith(ZIP_EXT))
                                {
                                    var str = inf.key.TrimEnd('.', 'z');
                                    var file = updater.remoteFiles.Find(data => data.key == str);
                                    if (file != null)
                                    {
                                        updater.total_unzip_bytes += file.size;
                                        _zip_files.Enqueue(inf);
                                    }
                                }
                            }
                        }
                        catch (Exception err)
                        {
                            log.Error(err);
                            needFiles.Enqueue(inf);
                        }
                    }
                    catch (Exception err)
                    {
                        needFiles.Enqueue(inf);
                        log.ErrorFormat(err.Message, err);
                        updater.QueueEvent(
                            new MPQUpdaterEvent(MPQUpdaterEvent.TYPE_ERROR,
                            string.Format("DOWNLOAD_ZIPS : {0} : {1}", inf.file, err.Message), err));
                        return true;
                    }
                }
                return false;
            }

            private bool run_download_single(RemoteFileInfo inf, long exist_size, long need_bytes)
            {
                long old_current_download_bytes = updater.current_download_bytes.Value;

                if (updater.driver.RunDownloadSingle(updater, inf, exist_size, need_bytes, updater.current_download_bytes) == false)
                {
                    return true;
                }
                if (!check_md5(inf.file, inf.md5)) return true;

                if ((old_current_download_bytes + need_bytes) != updater.current_download_bytes.Value)
                {
                    updater.current_download_bytes.Value = old_current_download_bytes + need_bytes;
                }
                return false;
            }

            private bool run_unzip()
            {
                while(_zip_files.Count > 0 )
                {
                    var inf = _zip_files.Dequeue();
                    if (updater.is_exit) return true;
                    var mpqName = inf.key.TrimEnd('.', 'z');
                    var info = updater.remoteFiles.Find(data => data.key == mpqName);

                    // 如果需要的空间无法满足
//                    long save_avaliable = updater.driver.GetAvaliableSpace(updater.save_root.FullName);
                    long save_avaliable = updater._mSaveAvaliable;
                    long need_bytes = info.size;
                    if (need_bytes >= save_avaliable)
                    {
                        _zip_files.Enqueue(inf);
                        updater.QueueEvent(new MPQUpdaterEvent(
                            MPQUpdaterEvent.TYPE_NOT_ENOUGH_SPACE, "Space not available!"));
                        return true;
                    }

                    // 开始解压缩
                    if (inf.file.Exists)
                    {
                        try
                        {
                            log.Info(string.Format("开始解压缩: {0} -> {1}", inf.key, info.file.Extension));
                            if (run_unzip_single(inf, info) || !info.file.Exists)
                            {
                                _zip_files.Enqueue(inf);
                            }
                            else
                            {
                                inf.file.Delete();
                            }
                        }
                        catch (Exception err)
                        {
                            log.Error(err);
                            _zip_files.Enqueue(inf);
                            throw;
                        }
                    }
                }

                return false;
            }

            private bool run_unzip_single(RemoteFileInfo inf, RemoteFileInfo mpqf)
            {
                long need_bytes = mpqf.size;
                long old_current_unzip_bytes = updater.current_unzip_bytes.Value;
                if (updater.driver.RunUnzipSingle(updater, inf, mpqf, updater.current_unzip_bytes) == false)
                {
                    return true;
                }
                if ((old_current_unzip_bytes + need_bytes) != updater.current_unzip_bytes.Value)
                {
                    updater.current_unzip_bytes.Value = old_current_unzip_bytes + need_bytes;
                }
                return false;
            }

            private bool check_md5(FileInfo ff, string md5)
            {
                string fmd5;
                if (updater.driver.RunGetFileMD5(ff.FullName, out fmd5) == false)
                {
                    return false;
                }
                if (fmd5.ToLower().Equals(md5.ToLower()))
                {
                    return true;
                }
                return false;
            }
        }

        //----------------------------------------------------------------------------------------------------------

        public void Dispose()
        {
            is_exit = true;
            try
            {
                if (workthread != null)
                {
                    workthread.Join();
                }
            }
            catch (System.Exception err)
            {
                log.Error(err.Message, err);
                //Console.WriteLine(e);
            }
        }


        public ICollection<FileInfo> GetAllFiles()
        {
            List<FileInfo> files = new List<FileInfo>();
            lock (remoteFiles)
            {
                foreach (RemoteFileInfo rf in remoteFiles)
                {
                    files.Add(rf.file);
                }
            }
            return files;
        }

        public int DownloadTimeoutSEC
        {
            get;
            set;
        }

        public string[] UrlRoots { get { return url_roots; } }
        public string UrlRoot { get { return url_roots[0]; } }

        public string VersionText
        {
            get { return version_text; }
        }

        public long TotalDownloadBytes
        {
            get { return total_download_bytes; }
        }
        public long CurrentDownloadBytes
        {
            get { return current_download_bytes.Value; }
        }
        public long CurrentDownloadSpeed
        {
            get { return current_download_speed_BPS; }
        }
        public long TotalUnzipBytes
        {
            get { return total_unzip_bytes; }
        }
        public long CurrentUnzipBytes
        {
            get { return current_unzip_bytes.Value; }
        }
        public long CurrentUnzipSpeed
        {
            get { return current_unzip_spped_BPS; }
        }
        public bool IsRunning
        {
            get { return is_running; }
        }

        public bool IsDisposing
        {
            get { return is_running && is_exit; }
        }

        public Status CurrentStatus { get { return status.Value; } }


        public bool IsLoadFinised { get; private set; }
        //-----------------------------------------------------------------------------------------------------------

        //-----------------------------------------------------------------------------------------------------------

        public class RemoteFileInfo
        {
            private FileInfo _file;

            public readonly string md5;
            public readonly string key;
            public readonly uint size;

            public FileInfo file
            {
                get { _file.Refresh(); return _file; }
            }

            internal RemoteFileInfo(string md5, uint size, string key, FileInfo file)
            {
                this.md5 = md5;
                this.key = key;
                this.size = size;
                this._file = file;
            }

            internal bool IsCompletion()
            {
                if (file.Exists && file.Length == size)
                {
                    return true;
                }
                return false;
            }

            internal long NeedLoadSize()
            {
                if (file.Exists) return (size - file.Length);
                else return size;
            }
        }
    }
}