前言
本来想对 libcurl 封装,使用协程来处理 http 的异步操作。
因此项目引入了 libcurl 和 libgo ,但是实现的时候,由于 libcurl 在域名解析的时候,出了问题导致,使用协程不成功,后来又用线程封装的。
由于没使用上协程,我读了一下 curl 的源码,想了解的话,可以看看 《curl 源码解析》文章。
curl 在解析 nds 的时候启用了线程,感觉在协程和线程切换时存在的问题。
协程只是想处理异步请求,因为请求只是一去一回,如果每次拿线程搞感觉浪费... 对于下载的话使用协程则意义不大了!
但第一战战败... 想法没和代码匹配上,后续在读一下 libgo 的源码,看不看能不能使用协程来处理异步问题。
这里还是用线程实现的。
环境搭建
需要引入的三方库,libcurl 、libgo 和 jsoncpp。
VS2017 搭建比较简单,不单独出文章了。
•libcurl 版本:curl-7.75.0•libgo 版本:v3.0•jsoncpp 版本:jsoncpp-1.8.3
看一下工程设置:

将 libcurl 和 libgo 以三方库的形式添加到工程中,嫌麻烦而把 jsoncpp 直接将代码添加到了 CurlProject 项目中。
将 jsoncpp 的头文件和库文件拷贝到 CurlProject 的代码中,但会编译不过,需要在 writer.h 头文件开头部分添加代码:
#if defined(GNUC)#pragma GCC diagnostic push#pragma GCC diagnostic ignored “-Wdeprecated-declarations”#elif defined(_MSC_VER)#pragma warning(disable : 4996)#endif
功能需求
先说一下功能需求,通过 HTTTP,使用 libcurl 进行上传、下载图片流程:
•读取本地图片,使用 http post 请求,将 图片发送到 服务器。•服务器 url 地址 :http://xxxx.xxxx.cn/H5/UnityApi/doUpload•上传文件参数名:img_file•发送成功后,响应:{"code":100,"url":"http://xxxx.xxxx.cn/xxxx/xxx/xx.jpg"} (json形式)•然后通过返回链接,使用 curl 下载图片
这里要感谢小白同学,为我搭建的 http 服务器,从而完成了对编写的代码的调试和测试。
设计与类图
设计类结构:

•HttpHelper 类:http 辅助类;记录全局的 http 请求和下载 list、请求和下载线程入口、生成 http 请求 handle 等。•RequestHelper 类:请求类实现,对 libcurl 的封装,是 HttpRequest 的内部类;•HttpRequest 类:对请求类进行同步异步的封装。•DownloadHelper 类:下载类实现,对 libcurl 的封装,是 HttpDownloader 的内部类;•HttpDownloader 类:对下载类进行同步异步的封装。•ImageRequestCallback 类:业务层类,处理图片请求回调。•ImageDownCallback 类:业务层类,处理图片下载回调。
需要提及的是,在下载类中,封装了线程池下载。
和单位的不同,单位使用的单线程处理,支持断点续传,就是将读取已下载文件位置,然后继续下载。
这里,通过不同的文件大小判断,进行文件分割,然后将各个分割部分交给多线程处理,更为复杂一些。
代码实现
通用代码
目录管理类:windows 下使用
DirectoryManager.h
/*******************************************************************************************vic.MINg 2019- 07-06VS中定位当前工程项目的根目录只在window下使用********************************************************************************************/#pragma once#include <windows.h>#include <string>using namespace std;class DirectoryManager{public:static string GetRootDirectory();static string GetExeDirectory();private://构造函数DirectoryManager() {};//获取当前项目根目录static void loadRootDir();//获取exe文件目录static void loadExeDir();//静态数组static char rootPath[MAX_PATH]; //根目录static char exePath[MAX_PATH]; //执行文件目录};
DirectoryManager.cpp
/*******************************************************************************************vic.MINg 2019- 07-06********************************************************************************************/#include "../include/DirectoryManager.h"//类的静态数组 初始化char DirectoryManager::rootPath[MAX_PATH] = "";char DirectoryManager::exePath[MAX_PATH] = "";string DirectoryManager::GetRootDirectory(){loadRootDir();string root = rootPath;/*for (char &c : root){if (c == '\\')c = '\/';}*/for (auto iter = root.begin(); iter != root.end(); iter++){if (*iter == '\\')*iter = '\/';}return root;}string DirectoryManager::GetExeDirectory(){loadExeDir();string exe = exePath;exe = exe.substr(0, exe.find_last_of('\\')+1);/*for (char &c : exe){if (c == '\\')c = '\/';}*/for (auto iter = exe.begin(); iter != exe.end(); iter++){if (*iter == '\\')*iter = '\/';}return exe;}//获取当前项目根目录void DirectoryManager::loadRootDir(){GetCurrentDirectory(MAX_PATH, rootPath);}//获取exe文件目录void DirectoryManager::loadExeDir(){GetModuleFileName(NULL, exePath, MAX_PATH);}
任务池模型
TaskPool.h
/*** 任务池模型,TaskPool.h*/#pragma once#include <thread>#include <mutex>#include <condition_variable>#include <list>#include <vector>#include <memory>#include <iostream>class Task{public:virtual void doIt(){std::cout << "handle a task..." << std::endl;}virtual ~Task(){//为了看到一个task的销毁,这里刻意补上其析构函数std::cout << "a task destructed..." << std::endl;}};class TaskPool final{friend class HttpDownloader;public:TaskPool();~TaskPool();TaskPool(const TaskPool& rhs) = delete;TaskPool& operator=(const TaskPool& rhs) = delete;public:void init(int threadNum = 4);void stop();void addTask(Task* task);void removeAllTasks();void runTaskPool() { m_bRunning = true; }bool isEmptyTask() { return m_taskList.empty(); }private:void threadFunc();private:std::list<std::shared_ptr<Task>> m_taskList;std::mutex m_mutexList;std::condition_variable m_cv;bool m_bRunning;std::vector<std::shared_ptr<std::thread>> m_threads;std::condition_variable m_finish_cv;std::mutex m_finish_mutex;};
TaskPool.cpp
#include "../include/TaskPool.h"TaskPool::TaskPool() : m_bRunning(false){}TaskPool::~TaskPool(){removeAllTasks();}void TaskPool::init(int threadNum/* = 5*/){if (threadNum <= 0)threadNum = 5;m_bRunning = true;for (int i = 0; i < threadNum; ++i){std::shared_ptr<std::thread> spThread;spThread.reset(new std::thread(std::bind(&TaskPool::threadFunc, this)));m_threads.push_back(spThread);}}void TaskPool::threadFunc(){std::shared_ptr<Task> spTask;while (true){std::unique_lock<std::mutex> guard(m_mutexList);while (m_taskList.empty()){if (!m_bRunning)break;//如果获得了互斥锁,但是条件不合适的话,pthread_cond_wait会释放锁,不往下执行。//当发生变化后,条件合适,pthread_cond_wait将直接获得锁。m_cv.wait(guard);}if (!m_bRunning)break;spTask = m_taskList.front();m_taskList.pop_front();if (spTask == NULL)continue;spTask->doIt();spTask.reset();m_finish_cv.notify_one(); // 线程下载完毕 通知完成}std::cout << "exit thread, threadID: " << std::this_thread::get_id() << std::endl;}void TaskPool::stop(){m_bRunning = false;m_cv.notify_all();//等待所有线程退出for (auto& iter : m_threads){if (iter->joinable())iter->join();}}void TaskPool::addTask(Task* task){std::shared_ptr<Task> spTask;spTask.reset(task);{std::lock_guard<std::mutex> guard(m_mutexList);//m_taskList.push_back(std::make_shared<Task>(task));m_taskList.push_back(spTask);std::cout << "add a Task." << std::endl;}m_cv.notify_one();}void TaskPool::removeAllTasks(){{std::lock_guard<std::mutex> guard(m_mutexList);for (auto& iter : m_taskList){iter.reset();}m_taskList.clear();}}
Http 封装代码
原子锁:
HttpLock.h
/*************************************************vic.MINg 2021/03/30原子锁************************************************/#pragma once#include <atomic>#include <memory>// 原子锁class HttpLock{public:HttpLock():m_flag_(false) { }void Lock(){bool expect = false;while (!m_flag_.compare_exchange_weak(expect, true)){expect = false; // 这里一定要将 expect 复原,执行失败时 expect 结果是未定的}}void UnLock(){m_flag_.store(false);}private:std::atomic<bool> m_flag_;};class DoHttpLock{public:DoHttpLock(std::shared_ptr<HttpLock> & lock): m_lock(lock){m_lock->Lock();}~DoHttpLock(){m_lock->UnLock();}private:std::shared_ptr<HttpLock> m_lock;};
HTTP 辅助类
HttpHelper.h
/*************************************************vic.MINg 2021/03/30HTTP 辅助类************************************************/#pragma once#include <curl/curl.h>#include <libgo.h>#include <atomic>#include <memory>#include <list>#include "HttpLock.h"#include "HttpRequest.h"#include "HttpDownloader.h"class HttpHelper {protected:HttpHelper();public:~HttpHelper();static HttpHelper& Instance() // 这里 Instance 只负责原子叠加{static HttpHelper the_single_instance;s_id++;return the_single_instance;}static void set_share_handle(CURL* curl_handle); // 如果 总是访问 同一个服务器 ,有必要设置 DNS共享static void RequestCoroutine(void* param); // HTTP请求 协程static void RequestThread(void* param); // HTTP请求 线程static size_t RetriveHeaderFunction(char *buffer, size_t size, size_t nitems, void *userdata); // 检索 头 函数static size_t RetriveContentFunction(char *ptr, size_t size, size_t nmemb, void *userdata); // 检索 内容 函数static void DownloadCoroutine(void* param); // 下载 协程static void DownloadThread(void* param); // 下载 线程static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata);static int progress_callback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow);static std::list< std::shared_ptr<HttpRequest::RequestHelper> > s_async_requests;static std::list< std::shared_ptr<HttpDownloader::DownloadHelper> > s_async_downloads;static std::atomic<int> s_id; // 原子计数static std::shared_ptr<HttpLock> s_request_lock;static std::shared_ptr<HttpLock> s_download_lock;static CURLSH* s_share_handle;static std::condition_variable s_request_cv;static std::condition_variable s_download_cv;};
HttpHelper.cpp
#include "../include/HttpHelper.h"#include <thread>std::list< std::shared_ptr<HttpRequest::RequestHelper> > HttpHelper::s_async_requests;std::list< std::shared_ptr<HttpDownloader::DownloadHelper> > HttpHelper::s_async_downloads;std::atomic<int> HttpHelper::s_id{0};std::shared_ptr<HttpLock> HttpHelper::s_request_lock(new HttpLock);std::shared_ptr<HttpLock> HttpHelper::s_download_lock(new HttpLock);CURLSH* HttpHelper::s_share_handle = nullptr;std::condition_variable HttpHelper::s_request_cv;std::condition_variable HttpHelper::s_download_cv;HttpHelper::HttpHelper(){curl_global_init(CURL_GLOBAL_DEFAULT);s_share_handle = curl_share_init();curl_share_setopt(s_share_handle, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);}HttpHelper::~HttpHelper(){curl_share_cleanup(s_share_handle);curl_global_cleanup();s_async_requests.clear();s_async_downloads.clear();}void HttpHelper::set_share_handle(CURL* curl_handle){curl_easy_setopt(curl_handle, CURLOPT_SHARE, s_share_handle);curl_easy_setopt(curl_handle, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 5);}void HttpHelper::RequestCoroutine(void* param){co_sleep(10);std::shared_ptr<HttpRequest::RequestHelper>* request = reinterpret_cast<std::shared_ptr<HttpRequest::RequestHelper>*>(param);if (request){(*request)->Perform();co_yield; // 请求以后 可以交给其他 协程处理//if ((*request)->SelfClose()){// DoHttpLock http_lock(s_request_lock);// HttpHelper::s_async_requests.remove(*request);}}}void HttpHelper::RequestThread(void* param){std::shared_ptr<HttpRequest::RequestHelper>* request = reinterpret_cast<std::shared_ptr<HttpRequest::RequestHelper>*>(param);if (request){(*request)->Perform();std::unique_lock < std::mutex > uniqlock((*request)->m_mutex);s_request_cv.wait(uniqlock, [request] { // 这里的判断条件为lambda表达式,判断请求是否关闭,未关闭返回false,否则返回trueif ((*request)->SelfClose()) // 调用wait函数return true;return false;});// std::this_thread::sleep_for(std::chrono::milliseconds(100));DoHttpLock http_lock(s_request_lock);HttpHelper::s_async_requests.remove(*request);uniqlock.unlock();}}size_t HttpHelper::RetriveHeaderFunction(char *buffer, size_t size, size_t nitems, void *userdata){std::string* receive_header = reinterpret_cast<std::string*>(userdata);if (receive_header && buffer){receive_header->append(reinterpret_cast<const char*>(buffer), size * nitems);}return nitems * size;}size_t HttpHelper::RetriveContentFunction(char *ptr, size_t size, size_t nmemb, void *userdata){std::string* receive_content = reinterpret_cast<std::string*>(userdata);if (receive_content && ptr){receive_content->append(reinterpret_cast<const char*>(ptr), size * nmemb);}return nmemb * size;}void HttpHelper::DownloadCoroutine(void* param){co_sleep(10);std::shared_ptr<HttpDownloader::DownloadHelper>* request = reinterpret_cast<std::shared_ptr<HttpDownloader::DownloadHelper>*>(param);if (request){(*request)->Perform();co_yield;if ((*request)->SelfClose()){DoHttpLock http_lock(s_download_lock);HttpHelper::s_async_downloads.remove(*request);}}}void HttpHelper::DownloadThread(void* param){std::shared_ptr<HttpDownloader::DownloadHelper>* request = reinterpret_cast<std::shared_ptr<HttpDownloader::DownloadHelper>*>(param);if (request){(*request)->Perform();std::unique_lock < std::mutex > uniqlock((*request)->m_mutex);s_download_cv.wait(uniqlock, [request] { // 这里的判断条件为lambda表达式,判断请求是否关闭,未关闭返回false,否则返回trueif ((*request)->SelfClose()) // 调用wait函数return true;return false;});//std::this_thread::sleep_for(std::chrono::milliseconds(100));DoHttpLock http_lock(s_download_lock);HttpHelper::s_async_downloads.remove(*request);uniqlock.unlock();}}size_t HttpHelper::write_callback(char *ptr, size_t size, size_t nmemb, void *userdata){HttpDownloader::DownloadHelper::ThreadChunk* thread_chunk = reinterpret_cast<HttpDownloader::DownloadHelper::ThreadChunk*>(userdata);if (thread_chunk->_download->m_is_cancel){return 0;}DoHttpLock http_lock(thread_chunk->_download->m_httplock);size_t written = 0;int real_size = size * nmemb;if (thread_chunk->_endidx > 0){if (thread_chunk->_startidx <= thread_chunk->_endidx){if (thread_chunk->_startidx + real_size > thread_chunk->_endidx){real_size = thread_chunk->_endidx - thread_chunk->_startidx + 1;}}}int seek_error = fseek(thread_chunk->_fp, thread_chunk->_startidx, SEEK_SET);if (seek_error != 0){perror("fseek");}else{written = fwrite(ptr, 1, real_size, thread_chunk->_fp);}thread_chunk->_download->m_downloaded_size += written;thread_chunk->_startidx += written;return written;}int HttpHelper::progress_callback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow){HttpDownloader::DownloadHelper::ThreadChunk* thread_chunk = reinterpret_cast<HttpDownloader::DownloadHelper::ThreadChunk*>(clientp);DoHttpLock http_lock(thread_chunk->_download->m_httplock);double total_size = thread_chunk->_download->m_total_size;double downloaded_size = thread_chunk->_download->m_downloaded_size;void* userdata = thread_chunk->_download->m_userdata;int callback_result = thread_chunk->_download->m_download_callback(total_size, downloaded_size, userdata);return callback_result;}
HTTP 客户端请求
HttpRequest.h
/*************************************************vic.MINg 2021/03/30http 客户端请求 curl 实现************************************************/#pragma once#include <string>#include <functional>#include <memory>#include <map>#include <curl/curl.h>#include <mutex>class HttpRequest{public:typedef enum {REQUEST_SYNC, // 同步REQUEST_ASYNC, // 异步}RequestType;typedef enum {REQUEST_OK,REQUEST_INVALID_OPT,REQUEST_PERFORM_ERROR,REQUEST_OPENFILE_ERROR,REQUEST_INIT_ERROR,}RequestResult;//int id, bool success, const std::string& datatypedef std::function<void(int, bool, const std::string&)> ResultCallback;friend class HttpHelper;HttpRequest();~HttpRequest() = default;int SetRetryTimes(int retry_times = s_kRetryCount);int SetRequestId(int id);int SetRequestTimeout(long time_out = 0);int SetRequestUrl(const std::string& url);int SetMovedUrl(bool get_moved_url); // 重定向 url (true 表示 重定向)int SetPostData(const std::string& message);int SetPostData(const void* data, unsigned int size);int SetPostFile(const std::string& flag_name, const std::string& file_path); // post 上传文件int SetRequestHeader(const std::map<std::string, std::string>& headers); // 请求头信息int SetRequestHeader(const std::string& header);int SetRequestProxy(const std::string& proxy, long proxy_port); // 代理int SetResultCallback(ResultCallback rc);void* PerformRequest(RequestType request_type);static void Close(void* request_handle);static bool GetHttpCode(void* request_handle, long* http_code);static bool GetReceiveHeader(void* request_handle, std::string* header);static bool GetReceiveContent(void* request_handle, std::string* receive);static bool GetErrorString(void* request_handle, std::string* error_string);protected:class RequestHelper {public:RequestHelper();~RequestHelper();friend class HttpRequest;friend class HttpHelper;int SetRetryTimes(int retry_times) { m_retry_times = retry_times; return REQUEST_OK; }int SetRequestTimeout(long time_out = 0);int SetRequestUrl(const std::string& url);int SetMovedUrl(bool get_moved_url);int SetPostData(const void* data, unsigned int size);int SetPostFile(const std::string& flag_name, const std::string& file_path);int SetRequestHeader(const std::string& header);int SetRequestProxy(const std::string& proxy, long proxy_port);int SetResultCallback(ResultCallback rc);int Perform();long GetHttpCode() { return m_http_code; }bool GetHeader(std::string* header);bool GetContent(std::string* receive);bool GetErrorString(std::string* error_string);bool SelfClose(void) { return m_close_self; }protected:void ReqeustResultDefault(int id, bool success, const std::string& data);private:CURL* m_curl_handle;curl_slist* m_http_headers;curl_httppost* m_file_post; // 上传文件 指针 执行后需要 清理int m_retry_times;int m_id;bool m_close_self;bool m_is_running;long m_http_code;std::string m_receive_content;std::string m_receive_header;std::string m_error_string;char* m_post_data;std::mutex m_mutex; // 互斥锁 ,通知条件变量 使用ResultCallback m_result_callback;};private:std::shared_ptr<RequestHelper> m_request_handle;static const int s_kRetryCount = 3; // 重试次数};
RequestHelper.cpp
#include "../include/HttpRequest.h"#include "../include/HttpHelper.h"HttpRequest::RequestHelper::RequestHelper(): m_curl_handle(nullptr), m_http_headers(nullptr), m_close_self(false), m_is_running(false), m_retry_times(HttpRequest::s_kRetryCount), m_http_code(0), m_post_data(nullptr), m_file_post(nullptr){m_result_callback = std::bind(&RequestHelper::ReqeustResultDefault, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);m_id = HttpHelper::s_id;m_curl_handle = curl_easy_init();HttpHelper::set_share_handle(m_curl_handle);}HttpRequest::RequestHelper::~RequestHelper(){if (m_curl_handle){curl_easy_cleanup(m_curl_handle);}if (m_http_headers){curl_slist_free_all(reinterpret_cast<curl_slist*>(m_http_headers));}if (m_post_data){delete m_post_data;m_post_data = nullptr;}}int HttpRequest::RequestHelper::SetRequestTimeout(long time_out){if (m_curl_handle){return curl_easy_setopt(m_curl_handle, CURLOPT_TIMEOUT, 0);}return CURLE_FAILED_INIT;}int HttpRequest::RequestHelper::SetRequestUrl(const std::string& url){if (m_curl_handle){if (url.substr(0, 5) == "https") // 处理加密得 HTTPS 协议{curl_easy_setopt(m_curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);curl_easy_setopt(m_curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);}curl_easy_setopt(m_curl_handle, CURLOPT_VERBOSE, 1L); //打开终端打印return curl_easy_setopt(m_curl_handle, CURLOPT_URL, url.c_str());}return CURLE_FAILED_INIT;}int HttpRequest::RequestHelper::SetMovedUrl(bool get_moved_url){if (m_curl_handle){if (get_moved_url) // 是否重定向{curl_easy_setopt(m_curl_handle, CURLOPT_MAXREDIRS, 5);return curl_easy_setopt(m_curl_handle, CURLOPT_FOLLOWLOCATION, 1L);}else{return curl_easy_setopt(m_curl_handle, CURLOPT_FOLLOWLOCATION, 0L);}}return CURLE_FAILED_INIT;}int HttpRequest::RequestHelper::SetPostData(const void* data, unsigned int size){if (m_curl_handle /*&& data && size > 0*/){CURLcode curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_POST, 1); // 设置 post 请求if (curl_code == CURLE_OK){if (m_post_data){delete m_post_data;m_post_data = nullptr;}if (size == 0){curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_POSTFIELDS, "");}else{m_post_data = new char[size];memcpy(m_post_data, data, size);curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_POSTFIELDS, m_post_data);}}if (curl_code == CURLE_OK){curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_POSTFIELDSIZE, size);}return curl_code;}return CURLE_FAILED_INIT;}int HttpRequest::RequestHelper::SetPostFile(const std::string& flag_name, const std::string& file_path){if (m_curl_handle /*&& data && size > 0*/){CURLcode curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_CUSTOMREQUEST, "POST");struct curl_httppost *FormPost = 0;struct curl_httppost *LastPost = 0;CURLFORMcode form_code = curl_formadd(&FormPost, &LastPost,CURLFORM_COPYNAME, flag_name.c_str(),CURLFORM_FILE, file_path.c_str(),CURLFORM_END);if (form_code != CURL_FORMADD_OK)return CURLE_HTTP_POST_ERROR;curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_HTTPPOST, FormPost);m_file_post = FormPost;return curl_code;}return CURLE_FAILED_INIT;}int HttpRequest::RequestHelper::SetRequestHeader(const std::string& header){if (m_curl_handle && header.empty() == false){m_http_headers = curl_slist_append(reinterpret_cast<curl_slist*>(m_http_headers), header.c_str());return m_http_headers ? CURLE_OK : CURLE_FAILED_INIT;}return CURLE_FAILED_INIT;}int HttpRequest::RequestHelper::SetRequestProxy(const std::string& proxy, long proxy_port){//CURLOPT_PROXYif (m_curl_handle){CURLcode curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_PROXYPORT, proxy_port);curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_PROXY, proxy.c_str());return curl_code;}return CURLE_FAILED_INIT;}int HttpRequest::RequestHelper::SetResultCallback(ResultCallback rc){m_result_callback = rc;return CURLE_OK;}void HttpRequest::RequestHelper::ReqeustResultDefault(int id, bool success, const std::string& data){//default request callback do nothing}int HttpRequest::RequestHelper::Perform(){if (m_curl_handle){CURLcode curl_code;if (m_http_headers){curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_HTTPHEADER, reinterpret_cast<curl_slist*>(m_http_headers));if (curl_code != CURLE_OK){return curl_code;}}m_is_running = true;m_receive_header.clear();m_receive_content.clear();//set force http redirectSetMovedUrl(true);curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_HEADERFUNCTION, HttpHelper::RetriveHeaderFunction);curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_HEADERDATA, &m_receive_header);curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_WRITEFUNCTION, HttpHelper::RetriveContentFunction);curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_WRITEDATA, &m_receive_content);curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_NOPROGRESS, 1);// 在多线程场景下应该设置CURLOPT_NOSIGNAL选项,因为在解析DNS出现超时的时候将会发生“糟糕”的情况。curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_NOSIGNAL, 1);curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_CONNECTTIMEOUT_MS, 0);curl_code = curl_easy_perform(m_curl_handle);if (curl_code == CURLE_OPERATION_TIMEDOUT){int retry_count = m_retry_times;while (retry_count > 0){curl_code = curl_easy_perform(m_curl_handle);if (curl_code != CURLE_OPERATION_TIMEDOUT) break;retry_count--;}}curl_easy_getinfo(m_curl_handle, CURLINFO_RESPONSE_CODE, &m_http_code);if (curl_code == CURLE_OK && m_http_code == 200){m_result_callback(m_id, true, m_receive_content);}else{const char* err_string = curl_easy_strerror(curl_code);m_error_string = err_string;curl_code = CURLE_HTTP_POST_ERROR;m_result_callback(m_id, false, m_error_string);}m_is_running = false;if (m_http_headers){curl_slist_free_all(reinterpret_cast<curl_slist*>(m_http_headers));m_http_headers = nullptr;}if (m_file_post){curl_formfree(m_file_post);m_file_post = nullptr;}return curl_code;}return CURLE_FAILED_INIT;}bool HttpRequest::RequestHelper::GetHeader(std::string* header){if (m_receive_header.empty()) return false;else if (header) *header = m_receive_header;return true;}bool HttpRequest::RequestHelper::GetContent(std::string* receive){if (m_receive_content.empty()) return false;else if (receive) *receive = m_receive_content;return true;}bool HttpRequest::RequestHelper::GetErrorString(std::string* error_string){if (m_error_string.empty()) return false;else if (error_string) *error_string = m_error_string;return true;}
HttpRequest.cpp
#include "../include/HttpRequest.h"#include "../include/HttpHelper.h"#include <libgo.h>HttpRequest::HttpRequest() : m_request_handle(new HttpRequest::RequestHelper){HttpHelper::Instance();}int HttpRequest::SetRetryTimes(int retry_times){if (m_request_handle){m_request_handle->SetRetryTimes(retry_times);return REQUEST_OK;}return REQUEST_INIT_ERROR;}int HttpRequest::SetRequestId(int id){if (m_request_handle){m_request_handle->m_id = id;return REQUEST_OK;}return REQUEST_INIT_ERROR;}int HttpRequest::SetRequestTimeout(long time_out){if (m_request_handle){if (m_request_handle->SetRequestTimeout(time_out) == CURLE_OK){return REQUEST_OK;}else{return REQUEST_INVALID_OPT;}}return REQUEST_INIT_ERROR;}int HttpRequest::SetRequestUrl(const std::string& url){if (m_request_handle){if (m_request_handle->SetRequestUrl(url) == CURLE_OK){return REQUEST_OK;}else{return REQUEST_INVALID_OPT;}}return REQUEST_INIT_ERROR;}int HttpRequest::SetMovedUrl(bool get_moved_url){if (m_request_handle){if (m_request_handle->SetMovedUrl(get_moved_url) == CURLE_OK){return REQUEST_OK;}else{return REQUEST_INVALID_OPT;}}return REQUEST_INIT_ERROR;}int HttpRequest::SetPostData(const std::string& message){return SetPostData(message.c_str(), message.size());}int HttpRequest::SetPostData(const void* data, unsigned int size){if (m_request_handle){if (m_request_handle->SetPostData(data, size) == CURLE_OK){return REQUEST_OK;}else{return REQUEST_INVALID_OPT;}}return REQUEST_INIT_ERROR;}int HttpRequest::SetPostFile(const std::string& flag_name, const std::string& file_path){if (m_request_handle){if (m_request_handle->SetPostFile(flag_name, file_path) == CURLE_OK){return REQUEST_OK;}else{return REQUEST_INVALID_OPT;}}return REQUEST_INIT_ERROR;}int HttpRequest::SetRequestHeader(const std::map<std::string, std::string>& headers){if (m_request_handle){for (auto it = headers.begin(); it != headers.end(); ++it){std::string header = it->first;header += ": ";header += it->second;if (m_request_handle->SetRequestHeader(header) != CURLE_OK){return REQUEST_INVALID_OPT;}}return REQUEST_OK;}return REQUEST_INIT_ERROR;}int HttpRequest::SetRequestHeader(const std::string& header){if (m_request_handle){if (m_request_handle->SetRequestHeader(header) == CURLE_OK){return REQUEST_OK;}else{return REQUEST_INVALID_OPT;}}return REQUEST_INIT_ERROR;}int HttpRequest::SetRequestProxy(const std::string& proxy, long proxy_port){if (m_request_handle){if (m_request_handle->SetRequestProxy(proxy, proxy_port) == CURLE_OK){return REQUEST_OK;}else{return REQUEST_INVALID_OPT;}}return REQUEST_INIT_ERROR;}int HttpRequest::SetResultCallback(ResultCallback rc){if (m_request_handle){m_request_handle->SetResultCallback(rc);return REQUEST_OK;}return REQUEST_INIT_ERROR;}void HttpRequest::Close(void* request_handle){std::shared_ptr<RequestHelper>* request = (reinterpret_cast<std::shared_ptr<RequestHelper> *>(request_handle));if (request == nullptr){return;}bool basync = false;DoHttpLock http_lock(HttpHelper::s_request_lock);for (auto it = HttpHelper::s_async_requests.begin(); it != HttpHelper::s_async_requests.end(); ++it){if ((*request) == *it){std::unique_lock < std::mutex > uniqlock((*request)->m_mutex);//HttpHelper::s_async_requests.remove(*request);(*request)->m_close_self = true;basync = true;HttpHelper::s_request_cv.notify_one();break;}}if (basync == false){//request->reset();}}void* HttpRequest::PerformRequest(RequestType request_type){if (m_request_handle){if (m_request_handle->m_is_running){return nullptr;}if (request_type == REQUEST_SYNC){m_request_handle->Perform();return &m_request_handle;}else if (request_type == REQUEST_ASYNC){DoHttpLock http_lock(HttpHelper::s_request_lock);HttpHelper::s_async_requests.push_back(m_request_handle);std::shared_ptr<RequestHelper>& request = HttpHelper::s_async_requests.back();// 协程调用// go co_scheduler(HttpHelper::s_coroutine_scheduler) [&request] {// HttpHelper::RequestCoroutine((void*)&request);// };std::thread request_thread(HttpHelper::RequestThread, (void*)&request);request_thread.detach();return &request;}return nullptr;}return nullptr;}bool HttpRequest::GetHttpCode(void* request_handle, long* http_code){std::shared_ptr<RequestHelper>* request = reinterpret_cast<std::shared_ptr<RequestHelper>*>(request_handle);std::unique_lock < std::mutex > uniqlock((*request)->m_mutex);if (request && http_code){*http_code = (*request)->GetHttpCode();return true;}return false;}bool HttpRequest::GetReceiveHeader(void* request_handle, std::string* header){std::shared_ptr<RequestHelper>* request = reinterpret_cast<std::shared_ptr<RequestHelper>*>(request_handle);std::unique_lock < std::mutex > uniqlock((*request)->m_mutex);if (request){return (*request)->GetHeader(header);}return false;}bool HttpRequest::GetReceiveContent(void* request_handle, std::string* receive){std::shared_ptr<RequestHelper>* request = reinterpret_cast<std::shared_ptr<RequestHelper>*>(request_handle);std::unique_lock < std::mutex > uniqlock((*request)->m_mutex);if (request){return (*request)->GetContent(receive);}return false;}bool HttpRequest::GetErrorString(void* request_handle, std::string* error_string){std::shared_ptr<RequestHelper>* request = reinterpret_cast<std::shared_ptr<RequestHelper>*>(request_handle);std::unique_lock < std::mutex > uniqlock((*request)->m_mutex);if (request){return (*request)->GetErrorString(error_string);}return false;}/////////////////////////////////////////////////////////////////////////////////////////////////////////
下载任务
DownloadTask.h
/*************************************************vic.MINg 2021/04/01下载任务************************************************/#include "TaskPool.h"#include "HttpDownloader.h"class DownloadTask : public Task{public:DownloadTask(void* param);virtual ~DownloadTask() = default;virtual void doIt();protected:private:HttpDownloader::DownloadHelper::ThreadChunk* m_thread_chunk = nullptr;};
DownloadTask.cpp
#include "../include/DownloadTask.h"#include "../include/HttpRequest.h"DownloadTask::DownloadTask(void* param){m_thread_chunk = reinterpret_cast<HttpDownloader::DownloadHelper::ThreadChunk*>(param);}void DownloadTask::doIt(){int down_code = HttpRequest::REQUEST_PERFORM_ERROR;if (m_thread_chunk != nullptr){down_code = m_thread_chunk->_download->DoDownload(m_thread_chunk);}}
HTTP 客户端下载
/*************************************************vic.MINg 2021/03/31http 客户端下载 curl 实现************************************************/#pragma once#include <string>#include <functional>#include <memory>#include "../include/HttpLock.h"#include "../include/TaskPool.h"class HttpDownloader{public:typedef enum {DOWN_SYNC,DOWN_ASYNC,}DownType;//double total_size, double downloaded_size, void* userdatatypedef std::function<int(double, double, void*)> ProgressCallback;//int id, bool success, const std::string& datatypedef std::function<void(int, bool, const std::string&)> ResultCallback;friend class HttpHelper;friend class DownloadTask;HttpDownloader();~HttpDownloader() = default;std::string GetFiePath(void) {return m_request_handle->m_file_path;}int SetRequestProxy(const std::string& proxy, long proxy_port);int SetRetryTimes(int retry_times = s_kRetryCount);int SetTimeout(long time_out = 0);int SetDownloadUrl(const std::string& url);int SetUserData(void* userdata);int SetRequestId(int id);int SetProgressCallback(ProgressCallback pc);int SetResultCallback(ResultCallback rc);int DownloadFile(const std::string& file_name, int thread_count = 4);void* StartDownload(DownType down_type);static bool CancelDownload(void* handle);static void Close(void* handle);static bool GetHttpCode(void* handle, long* http_code);static bool GetReceiveHeader(void* handle, std::string* header);static bool GetErrorString(void* handle, std::string* error_string);static void* GetUserData(void* handle);protected:class DownloadHelper {public:typedef struct tThreadChunk{FILE* _fp;long _startidx;long _endidx;DownloadHelper* _download;}ThreadChunk;DownloadHelper();~DownloadHelper();friend class HttpDownloader;friend class HttpHelper;friend class DownloadTask;friend ThreadChunk;void SetRetryTimes(int retry_times) { m_retry_times = retry_times; }void SetRequestId(int id) { m_id = id; }int SetTimeout(long time_out = 0);int SetRequestUrl(const std::string& url);int SetRequestProxy(const std::string& proxy, long proxy_port);void SetUserData(void *userdata) { m_userdata = userdata; }int SetProgressCallback(ProgressCallback pc);int SetResultCallback(ResultCallback rc);int SetDownloadFile(const std::string& file_name);int SetDownloadThreadCount(int thread_count);int Perform();int GetHttpCode() { return m_http_code; }bool GetHeader(std::string* header);bool GetErrorString(std::string* error_string);bool SelfClose(void) { return m_close_self; }void* GetUserData(void) { return m_userdata; }protected:int DownloadDefaultCallback(double total_size, double downloaded_size, void* userdata);void ResultDefaultCallback(int id, bool success, const std::string& data);double GetDownloadFileSize();int DoDownload(ThreadChunk* thread_chunk);int SplitDownloadCount(double down_size);void Reset(void);private:int m_retry_times;int m_thread_count;int m_id;long m_time_out;std::string m_file_path;std::string m_url;std::string m_http_proxy;std::string m_receive_header;std::string m_error_string;bool m_close_self;bool m_multi_download;bool m_download_fail;bool m_is_running;bool m_is_cancel;void* m_userdata;long m_http_code;long m_proxy_port;double m_total_size;double m_downloaded_size;std::mutex m_mutex; // 互斥锁 ,通知条件变量 使用TaskPool m_threadPool_downloader; // 下载线程池std::shared_ptr<HttpLock> m_httplock;ProgressCallback m_download_callback;ResultCallback m_result_callback;};private:std::shared_ptr<DownloadHelper> m_request_handle;static const int s_kRetryCount = 3;static const int s_kThreadCount = 4;};
DownloadHelper.cpp
#include "../include/HttpDownloader.h"#include "../include/HttpHelper.h"#include "../include/DownloadTask.h"#include <stdio.h>#include <regex>#include <sstream>HttpDownloader::DownloadHelper::DownloadHelper(): m_close_self(false), m_retry_times(HttpDownloader::s_kRetryCount), m_thread_count(HttpDownloader::s_kThreadCount), m_http_code(0), m_time_out(0), m_proxy_port(0), m_total_size(0.0), m_downloaded_size(0.0), m_multi_download(false), m_download_fail(true), m_is_running(false), m_httplock(new HttpLock), m_userdata(NULL){m_download_callback = std::bind(&DownloadHelper::DownloadDefaultCallback, this,std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);m_result_callback = std::bind(&DownloadHelper::ResultDefaultCallback, this,std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);m_id = HttpHelper::s_id;}HttpDownloader::DownloadHelper::~DownloadHelper(){}int HttpDownloader::DownloadHelper::SetTimeout(long time_out /* = 0 */){m_time_out = time_out;return CURLE_OK;}int HttpDownloader::DownloadHelper::SetRequestUrl(const std::string& url){m_url = url;return CURLE_OK;}int HttpDownloader::DownloadHelper::SetRequestProxy(const std::string& proxy, long proxy_port){m_http_proxy = proxy;m_proxy_port = proxy_port;return CURLE_OK;}int HttpDownloader::DownloadHelper::SetProgressCallback(ProgressCallback pc){m_download_callback = pc;return CURLE_OK;}int HttpDownloader::DownloadHelper::SetResultCallback(ResultCallback rc){m_result_callback = rc;return CURLE_OK;}int HttpDownloader::DownloadHelper::SetDownloadFile(const std::string& file_name){m_file_path = file_name;return CURLE_OK;}int HttpDownloader::DownloadHelper::SetDownloadThreadCount(int thread_count){m_thread_count = thread_count;return CURLE_OK;}int HttpDownloader::DownloadHelper::Perform(){m_total_size = GetDownloadFileSize();if (m_total_size < 0){return HttpRequest::REQUEST_PERFORM_ERROR;}std::string out_file_name = m_file_path;std::string src_file_name = out_file_name;out_file_name += ".tmp";FILE *fp = nullptr;remove(out_file_name.c_str());remove(src_file_name.c_str());fp = fopen(out_file_name.c_str(), "wb");if (!fp){return HttpRequest::REQUEST_OPENFILE_ERROR;}int down_code = HttpRequest::REQUEST_PERFORM_ERROR;int thread_count = SplitDownloadCount(m_total_size);thread_count = 3;m_thread_count = thread_count > m_thread_count ? m_thread_count : thread_count;//文件大小有分开下载的必要并且服务器支持多线程下载时,启用多线程下载if (m_multi_download && m_thread_count > 1){long gap = static_cast<long>(m_total_size) / m_thread_count;m_threadPool_downloader.init(m_thread_count); // m_thread_count 个线程for (int i = 0; i < m_thread_count; i++){ThreadChunk* thread_chunk = new ThreadChunk;thread_chunk->_fp = fp;thread_chunk->_download = this;if (i < m_thread_count - 1){thread_chunk->_startidx = i * gap;thread_chunk->_endidx = thread_chunk->_startidx + gap - 1;}else{thread_chunk->_startidx = i * gap;thread_chunk->_endidx = -1;}// 下载块任务 添加到线程池中DownloadTask* pTask = new DownloadTask((void*)thread_chunk);m_threadPool_downloader.addTask(pTask);}std::unique_lock < std::mutex > uniqlock(m_threadPool_downloader.m_finish_mutex);//m_threadPool_downloader.runTaskPool();m_threadPool_downloader.m_finish_cv.wait(uniqlock, [this] {if (m_threadPool_downloader.isEmptyTask())return true;return false;});std::this_thread::sleep_for(std::chrono::milliseconds(50));m_threadPool_downloader.removeAllTasks();m_threadPool_downloader.stop();}else{ThreadChunk* thread_chunk = new ThreadChunk;thread_chunk->_fp = fp;thread_chunk->_download = this;thread_chunk->_startidx = 0;thread_chunk->_endidx = 0;down_code = DoDownload(thread_chunk);std::this_thread::sleep_for(std::chrono::milliseconds(50));}fclose(fp);if (m_download_fail == false){rename(out_file_name.c_str(), src_file_name.c_str()); // 下载完成 重命名文件名}else{remove(out_file_name.c_str()); // 删除 临时文件}m_result_callback(m_id, m_download_fail ? false : true, m_error_string);m_is_running = false;return down_code;}bool HttpDownloader::DownloadHelper::GetHeader(std::string* header){if (m_receive_header.empty()) return false;else if (header) *header = m_receive_header;return true;}bool HttpDownloader::DownloadHelper::GetErrorString(std::string* error_string){if (m_error_string.empty()) return false;else if (error_string) *error_string = m_error_string;return true;}int HttpDownloader::DownloadHelper::DownloadDefaultCallback(double total_size, double downloaded_size, void* userdata){return 0;}void HttpDownloader::DownloadHelper::ResultDefaultCallback(int id, bool success, const std::string& data){}double HttpDownloader::DownloadHelper::GetDownloadFileSize(){if (m_url.empty()){return -1.0;}else{double down_file_length = -1.0;CURL *handle = curl_easy_init();HttpHelper::set_share_handle(handle);if (handle){curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str());curl_easy_setopt(handle, CURLOPT_HEADER, 1);curl_easy_setopt(handle, CURLOPT_NOBODY, 1);curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 5);curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, HttpHelper::RetriveHeaderFunction);curl_easy_setopt(handle, CURLOPT_HEADERDATA, &m_receive_header);curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, HttpHelper::RetriveContentFunction);curl_easy_setopt(handle, CURLOPT_WRITEDATA, NULL);curl_easy_setopt(handle, CURLOPT_RANGE, "2-");CURLcode curl_code = curl_easy_perform(handle);if (curl_code == CURLE_OPERATION_TIMEDOUT){int retry_count = m_retry_times;while (retry_count > 0){curl_code = curl_easy_perform(handle);if (curl_code != CURLE_OPERATION_TIMEDOUT) break;retry_count--;}}curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &m_http_code);if (curl_code == CURLE_OK){curl_easy_getinfo(handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &down_file_length);//匹配"Content-Range: bytes 2-1449/26620" 则证明支持多线程下载std::regex pattern("CONTENT-RANGE\\s*:\\s*\\w+\\s*(\\d+)-(\\d*)/(\\d+)", std::regex::icase);m_multi_download = std::regex_search(m_receive_header, pattern);}else{const char* err_string = curl_easy_strerror(curl_code);m_error_string = err_string;m_result_callback(m_id, false, m_error_string);}curl_easy_cleanup(handle);}return down_file_length;}}int HttpDownloader::DownloadHelper::DoDownload(ThreadChunk* thread_chunk){CURL* curl_handle = curl_easy_init();HttpHelper::set_share_handle(curl_handle);if (thread_chunk->_download->m_url.substr(0, 5) == "https"){curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);}curl_easy_setopt(curl_handle, CURLOPT_URL, thread_chunk->_download->m_url.c_str());const char* user_agent = ("Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0");// const char* user_agent = ("Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:28.0) Gecko/20100101 Firefox/28.0");curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent);curl_easy_setopt(curl_handle, CURLOPT_MAXREDIRS, 5L);curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L);curl_easy_setopt(curl_handle, CURLOPT_POST, 0L);curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT_MS, 0L);curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, thread_chunk->_download->m_time_out); //0 means block alwayscurl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, HttpHelper::write_callback);curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, thread_chunk);curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, HttpHelper::RetriveHeaderFunction);curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, NULL);curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L);curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, HttpHelper::progress_callback);curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, thread_chunk);curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, 1L);curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, 5L);if (thread_chunk->_endidx != 0){std::string down_range;std::ostringstream ostr;if (thread_chunk->_endidx > 0){ostr << thread_chunk->_startidx << "-" << thread_chunk->_endidx;}else{ostr << thread_chunk->_startidx << "-";}down_range = ostr.str();curl_easy_setopt(curl_handle, CURLOPT_RANGE, down_range.c_str());}CURLcode curl_code = curl_easy_perform(curl_handle);if (curl_code == CURLE_OPERATION_TIMEDOUT){int retry_count = m_retry_times;while (retry_count > 0){curl_code = curl_easy_perform(curl_handle);if (curl_code != CURLE_OPERATION_TIMEDOUT) break;retry_count--;}}long http_code;curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);if (curl_code == CURLE_OK && (http_code >= 200 && http_code <= 300)){m_http_code = http_code;thread_chunk->_download->m_download_fail = false;}else{const char* err_string = curl_easy_strerror(curl_code);m_error_string = err_string;thread_chunk->_download->m_download_fail = true;m_http_code = http_code;}curl_easy_cleanup(curl_handle);delete thread_chunk;return curl_code;}int HttpDownloader::DownloadHelper::SplitDownloadCount(double down_size){const double size_2mb = 2.0 * 1024 * 1024;const double size_10mb = 10.0 * 1024 * 1024;const double size_50mb = 50.0 * 1024 * 1024;if (down_size <= size_2mb){return 1;}else if (down_size > size_2mb && down_size <= size_10mb){return static_cast<int>(down_size / (size_2mb));}else if (down_size > size_10mb && down_size <= size_50mb){return HttpDownloader::s_kThreadCount + 1;}else{int down_count = static_cast<int>(down_size / size_10mb);return down_count > 10 ? 10 : down_count;}return 1;}void HttpDownloader::DownloadHelper::Reset(){if (m_is_running){return;}m_close_self = false;m_multi_download = false;m_download_fail = true;m_is_running = false;m_is_cancel = false;m_http_code = 0;m_total_size = 0.0;m_downloaded_size = 0.0;m_receive_header = "";m_error_string = "";}
HttpDownloader.cpp
#include "../include/HttpDownloader.h"#include "../include/HttpHelper.h"#include <libgo.h>HttpDownloader::HttpDownloader():m_request_handle(new HttpDownloader::DownloadHelper){HttpHelper::Instance();}int HttpDownloader::SetRequestProxy(const std::string& proxy, long proxy_port){if (m_request_handle){if (m_request_handle->SetRequestProxy(proxy, proxy_port) == CURLE_OK){return 0;}else{return HttpRequest::REQUEST_INVALID_OPT;}}return HttpRequest::REQUEST_INIT_ERROR;}int HttpDownloader::SetRetryTimes(int retry_times /* = s_kRetryCount */){if (m_request_handle){m_request_handle->SetRetryTimes(retry_times);return HttpRequest::REQUEST_OK;}return HttpRequest::REQUEST_INIT_ERROR;}int HttpDownloader::SetTimeout(long time_out /* = 0 */){if (m_request_handle){if (m_request_handle->SetTimeout(time_out) == CURLE_OK){return HttpRequest::REQUEST_OK;}else{return HttpRequest::REQUEST_INVALID_OPT;}}return HttpRequest::REQUEST_INIT_ERROR;}int HttpDownloader::SetDownloadUrl(const std::string& url){if (m_request_handle){if (m_request_handle->SetRequestUrl(url) == CURLE_OK){return HttpRequest::REQUEST_OK;}else{return HttpRequest::REQUEST_INVALID_OPT;}}return HttpRequest::REQUEST_INIT_ERROR;}int HttpDownloader::SetUserData(void* userdata){if (m_request_handle){m_request_handle->SetUserData(userdata);return HttpRequest::REQUEST_OK;}return HttpRequest::REQUEST_INIT_ERROR;}int HttpDownloader::SetRequestId(int id){if (m_request_handle){m_request_handle->SetRequestId(id);return HttpRequest::REQUEST_OK;}return HttpRequest::REQUEST_INIT_ERROR;}int HttpDownloader::SetProgressCallback(ProgressCallback pc){if (m_request_handle){m_request_handle->SetProgressCallback(pc);return HttpRequest::REQUEST_OK;}return HttpRequest::REQUEST_INIT_ERROR;}int HttpDownloader::SetResultCallback(ResultCallback rc){if (m_request_handle){m_request_handle->SetResultCallback(rc);return HttpRequest::REQUEST_OK;}return HttpRequest::REQUEST_INIT_ERROR;}int HttpDownloader::DownloadFile(const std::string& file_name, int thread_count /* = 5 */){if (m_request_handle){m_request_handle->SetDownloadFile(file_name);m_request_handle->SetDownloadThreadCount(thread_count);}return HttpRequest::REQUEST_INIT_ERROR;}void* HttpDownloader::StartDownload(DownType down_type){if (m_request_handle){if (m_request_handle->m_is_running){return nullptr;}m_request_handle->Reset();if (down_type == DOWN_SYNC){m_request_handle->Perform();return &m_request_handle;}else if (down_type == DOWN_ASYNC){DoHttpLock http_lock(HttpHelper::s_download_lock);HttpHelper::s_async_downloads.push_back(m_request_handle);std::shared_ptr<DownloadHelper>& request = HttpHelper::s_async_downloads.back();// 协程调用//go co_scheduler(HttpHelper::s_coroutine_scheduler) [request] {// HttpHelper::DownloadCoroutine((void*)&request);//};std::thread download_thread(HttpHelper::DownloadThread, (void*)&request);download_thread.detach();return &request;}return nullptr;}return nullptr;}void HttpDownloader::Close(void* handle){std::shared_ptr<DownloadHelper>* request = (reinterpret_cast<std::shared_ptr<DownloadHelper> *>(handle));if ( request == nullptr){return;}bool basync = false;DoHttpLock http_lock(HttpHelper::s_download_lock);for (auto it = HttpHelper::s_async_downloads.begin(); it != HttpHelper::s_async_downloads.end(); ++it){if ((*request) == *it){std::unique_lock < std::mutex > uniqlock((*request)->m_mutex);//HttpHelper::s_async_downloads.remove(*request);(*request)->m_close_self = true;basync = true;HttpHelper::s_download_cv.notify_one();break;}}if (basync == false){//(*request)->m_is_cancel = true;//request->reset();}}bool HttpDownloader::CancelDownload(void* handle){std::shared_ptr<DownloadHelper>* request = (reinterpret_cast<std::shared_ptr<DownloadHelper> *>(handle));if (request == nullptr){return false;}(*request)->m_is_cancel = true;return true;}bool HttpDownloader::GetHttpCode(void* handle, long* http_code){std::shared_ptr<DownloadHelper>* request = reinterpret_cast<std::shared_ptr<DownloadHelper>*>(handle);if (request && http_code){*http_code = (*request)->GetHttpCode();return true;}return false;}bool HttpDownloader::GetErrorString(void* handle, std::string* error_string){std::shared_ptr<DownloadHelper>* request = reinterpret_cast<std::shared_ptr<DownloadHelper>*>(handle);if (request){return (*request)->GetErrorString(error_string);}return false;}bool HttpDownloader::GetReceiveHeader(void* handle, std::string* header){std::shared_ptr<DownloadHelper>* request = reinterpret_cast<std::shared_ptr<DownloadHelper>*>(handle);if (request){return (*request)->GetHeader(header);}return false;}void* HttpDownloader::GetUserData(void* handle){std::shared_ptr<DownloadHelper>* request = reinterpret_cast<std::shared_ptr<DownloadHelper>*>(handle);if (request){return (*request)->GetUserData();}return nullptr;}//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
业务代码
HttpImageFlow.h
/*************************************************vic.MINg 2021/04/01HTTP 图片 流程 类************************************************//**************************************************************************************************************/* HTTP 图片 流程/* 读取本地图片,使用 http post 请求,将 图片发送到 服务器,/* 服务器 url 地址 :http://xxxx.xxxx.cn/H5/UnityApi/doUpload/* 上传文件参数名:img_file/* 发送成功后,响应:{"code":100,"url":"http://xxxx.xxxx.cn/xxxx/xxx/xx.jpg"} (json形式)/* 然后通过返回链接,使用 curl 下载图片**************************************************************************************************************/#include "HttpDownloader.h"#include "HttpRequest.h"#define URL_UPLOAD_IMAGE "http://xxxx.xxxx.cn/H5/UnityApi/doUpload" // 上传文件URL#define UPLOAD_IMAGE_FOLDER "upload_img/" // 上传图片 文件夹#define DOWNLOAD_IMAGE_FOLDER "download_img/" // 下载图片 文件夹#define POST_FLAG_NAME "img_file"class ImageDownCallback{public:ImageDownCallback(std::shared_ptr<HttpDownloader> downloader):m_down_finished(false), m_download_helper(nullptr),m_request_handle(downloader) {}virtual ~ImageDownCallback(){m_download_helper = nullptr;}void DownResultCallback(int id, bool success, const std::string& data);int DownProgressCallback(double total_size, double downloaded_size, void* userdata);bool IsDownFinished(void) { return m_down_finished; }void* m_download_helper;private:bool m_down_finished;std::shared_ptr<HttpDownloader> m_request_handle;};class ImageRequestCallback{public:ImageRequestCallback(std::shared_ptr<HttpRequest> request):m_request_finished(false), m_request_helper(nullptr),m_request_handle(request) {}virtual ~ImageRequestCallback(){m_request_helper = nullptr;}void RequestResultCallback(int id, bool success, const std::string& data);bool IsRequestFinish(void) { return m_request_finished; }void* m_request_helper;private:bool m_request_finished;std::shared_ptr<HttpRequest> m_request_handle;};////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////void HttpImageFlow(std::string image);// 封装函数 post 请求 上传服务器void ImageUploadFlow(const std::string& upload_url, const std::string& img_name);// 封装函数 下载 图片void ImageDownFlow(const std::string& download_url);////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// Malloc_FileToBuffer 和 Free_FileToBuffer 必须 配套使用,封装 Free_FileToBuffer 就是为了防止内存泄漏// 一次将整个文件读入内存 申请内存Bufferint Malloc_FileToBuffer(const std::string& file_path, char* &buffer, long& size);// 释放读取文件时申请的Buffervoid Free_FileToBuffer(char* buffer);// 把字符串的 strsrc 替换成 strdstvoid string_replace(std::string &strBig, const std::string &strsrc, const std::string &strdst);// 获取路径或URL的文件名(包括后缀,如 C:\Test\abc.xyz --> abc.xyz)std::string GetPathOrURLShortName(std::string strFullName);int Post_Data_By_LibCurl(const char *file_path, const char *filename);
HttpImageFlow.cpp
#include "../include/HttpImageFlow.h"#include "../include/HttpDownloader.h"#ifdef _WIN32#include "../include/DirectoryManager.h"#endif#include "jsoncpp/json.h"#include <libgo.h>#include <iostream>#include <fstream>void ImageDownCallback::DownResultCallback(int id, bool success, const std::string& data){printf(data.c_str());if (success){}// 下载完成后 关闭请求HttpDownloader::Close(&m_request_handle);m_down_finished = true;}int ImageDownCallback::DownProgressCallback(double total_size, double downloaded_size, void* userdata){long tmp = static_cast<long>(downloaded_size / total_size * 100);// if (m_request_handle)// printf("%s down progress %d \n", m_request_handle->GetFiePath().c_str() ,tmp)//printf("down progress %d \n", tmp);return 0;}void ImageRequestCallback::RequestResultCallback(int id, bool success, const std::string& data){printf(data.c_str());if (success && m_request_finished == false){if (m_request_handle && m_request_helper){long http_code;if (m_request_handle->GetHttpCode(m_request_helper, &http_code))std::cout << "http code: " << http_code << std::endl;std::string header;if (m_request_handle->GetReceiveHeader(m_request_helper, &header)){std::cout << header << std::endl;}std::string content;if (m_request_handle->GetReceiveContent(m_request_helper, &content)){// Json 解析Json::Reader reader;Json::Value value;if (reader.parse(content, value)){int code = value["code"].asInt();if (code == 100){std::string download_url = value["url"].asString();//download_url = "http://xxxx.xxxx.cn/Data/UploadFiles/unity_video/2021-04-08/606eaf3a80f58.jpg";// 图片下载流程ImageDownFlow(download_url);}}}}}HttpRequest::Close(m_request_helper);m_request_finished = true;}//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////void string_replace(std::string &strBig, const std::string &strsrc, const std::string &strdst){std::string::size_type pos = 0;std::string::size_type srclen = strsrc.size();std::string::size_type dstlen = strdst.size();while ((pos = strBig.find(strsrc, pos)) != std::string::npos){strBig.replace(pos, srclen, strdst);pos += dstlen;}}std::string GetPathOrURLShortName(std::string strFullName){if (strFullName.empty()){return "";}string_replace(strFullName, "/", "\\");std::string::size_type iPos = strFullName.find_last_of('\\') + 1;return strFullName.substr(iPos, strFullName.length() - iPos);}void ImageDownFlow(const std::string& download_url){std::shared_ptr<HttpDownloader> image_downloader = std::make_shared<HttpDownloader>();std::shared_ptr<ImageDownCallback> imagedown_callback = std::make_shared<ImageDownCallback>(image_downloader);image_downloader->SetDownloadUrl(download_url.c_str());image_downloader->SetProgressCallback(std::bind(&ImageDownCallback::DownProgressCallback, imagedown_callback.get(), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));image_downloader->SetResultCallback(std::bind(&ImageDownCallback::DownResultCallback, imagedown_callback.get(), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));#ifdef _WIN32std::string down_filepath = DirectoryManager::GetExeDirectory() + DOWNLOAD_IMAGE_FOLDER + GetPathOrURLShortName(download_url);#elsestd::string down_filepath = DOWNLOAD_IMAGE_FOLDER + GetPathOrURLShortName(download_url);#endifimage_downloader->DownloadFile(down_filepath);imagedown_callback->m_download_helper = image_downloader->StartDownload(HttpDownloader::DOWN_ASYNC);}void ImageUploadFlow(const std::string& upload_url, const std::string& img_name){std::shared_ptr<HttpRequest> image_request = std::make_shared<HttpRequest>();std::shared_ptr<ImageRequestCallback> imageupload_callback = std::make_shared<ImageRequestCallback>(image_request);long image_size = 0;char* image_buffer = nullptr;#ifdef _WIN32std::string image_filepath = DirectoryManager::GetExeDirectory() + UPLOAD_IMAGE_FOLDER + img_name;#elsestd::string image_filepath = UPLOAD_IMAGE_FOLDER + img_name;#endif//printf(image_filepath.c_str());//if (Malloc_FileToBuffer(image_filepath, image_buffer, image_size) == 0){image_request->SetRequestUrl(upload_url);image_request->SetResultCallback(std::bind(&ImageRequestCallback::RequestResultCallback, imageupload_callback.get(), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));//image_request->SetRequestHeader("Content-Type: application/octet-stream");image_request->SetPostFile(POST_FLAG_NAME, image_filepath);imageupload_callback->m_request_helper = image_request->PerformRequest(HttpRequest::REQUEST_ASYNC);//Free_FileToBuffer(image_buffer);while (imageupload_callback->IsRequestFinish() == false){co_sleep(100);co_yield;}co_sleep(200);std::cout << "ImageUploadFlow RequestFinish ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" << std::endl;}}int Malloc_FileToBuffer(const std::string& file_path, char* &buffer, long& size){/* 若要一个byte不漏地读入整个文件,只能采用二进制方式打开 */FILE * pFile = fopen(file_path.c_str(), "rb");if (pFile == NULL) return -1;/* 获取文件大小 */fseek(pFile, 0, SEEK_END);size = ftell(pFile);rewind(pFile);/* 分配内存存储整个文件 */buffer = (char*)malloc(sizeof(char)*size);if (buffer == NULL) return -2;/* 将文件拷贝到buffer中 */size_t result = fread(buffer, 1, size, pFile);if (result != size) return -3;/* 结束演示,关闭文件并释放内存 */fclose(pFile);return 0;}void Free_FileToBuffer(char* buffer){if (buffer) { free(buffer); buffer = nullptr; }}void HttpImageFlow(std::string image){ImageUploadFlow(URL_UPLOAD_IMAGE, image);}/////////////////////////////////////////////////////////////////////////////////////////////////////size_t write_callback(void *buffer, size_t size, size_t nmemb, FILE *file) {//size_t r_size = fwrite(buffer, size, nmemb, file);return 0;}int Post_Data_By_LibCurl(const char *file_path, const char *filename){char url[256] = { 0 };struct curl_httppost *FormPost = 0;struct curl_httppost *LastPost = 0;sprintf(url, URL_UPLOAD_IMAGE);CURL *CurlHandle = curl_easy_init();curl_easy_setopt(CurlHandle, CURLOPT_CUSTOMREQUEST, "POST");curl_easy_setopt(CurlHandle, CURLOPT_URL, url);curl_easy_setopt(CurlHandle, CURLOPT_VERBOSE, 1L);//打开终端打印curl_easy_setopt(CurlHandle, CURLOPT_WRITEFUNCTION, write_callback);//设置回调函数curl_easy_setopt(CurlHandle, CURLOPT_WRITEDATA, NULL);//设置回调函数的参数,获取反馈信息curl_formadd(&FormPost, &LastPost, CURLFORM_COPYNAME, "img_file", CURLFORM_FILE, file_path, CURLFORM_END);curl_easy_setopt(CurlHandle, CURLOPT_HTTPPOST, FormPost);CURLcode nRet = curl_easy_perform(CurlHandle);if (nRet == CURLE_OK) {printf("perform successful\n");}else{printf("perform failed\n");}curl_formfree(FormPost);curl_easy_cleanup(CurlHandle);return nRet;}
测试代码
#include "../include/HttpImageFlow.h"#include <iostream>#include <libgo.h>#include <Winsock2.h>#pragma comment(lib, "ws2_32.lib")int main(int argc, char** argv){//std::string path = "D:/Back/CoolWorm/C_LearnFoerver/C00_Programme/Vs2017/CurlProject/x64/Debug/upload_img/assassin.jpg";//std::string name = "assassin.jpg";//Post_Data_By_LibCurl(path.c_str(), name.c_str());co::Scheduler* sched = co::Scheduler::Create();// 启动2个线程执行新创建的调度器std::thread thr([sched] { sched->Start(2); });thr.detach();go co_scheduler(sched) [] {// HttpImageFlow("assassin.jpg");for (int i = 0; i < 10; i++){HttpImageFlow("assassin.jpg");HttpImageFlow("magician.jpg");HttpImageFlow("soldier.jpg");}};// 测试// 在新创建的调度器上创建一个协程go co_scheduler(sched) [] {printf("run in the scheduler.\n");};getchar();sched->Stop();return 0;}




