seh*_*ehe 5 c++ performance boost boost-spirit boost-spirit-qi
我想提出一个刚刚把我送到兔子洞的主题并提出一个关于qi :: symbols的问题.
这一切都是在我查看新的野兽库并阅读教程示例时开始的
它从一个从http路径扩展中猜出mime类型的函数开始.我开始仔细观察并看到了这个:
auto const ext = [&path]
{
auto const pos = path.rfind(".");
if(pos == boost::beast::string_view::npos)
return boost::beast::string_view{};
return path.substr(pos);
}();
Run Code Online (Sandbox Code Playgroud)
我花了一段时间才发现它是C++风格的IIFE,并且ext在声明它不变时用于初始化.
无论如何,我开始测试是否会产生任何性能差异,这将证明可怕的可读性与直接实现的合理性.
这样做我开始想知道这在qi :: symbols中实现得不是更好.所以我想出了两个替代实现:
#include <boost/smart_ptr/scoped_array.hpp>
#include <boost/accumulators/accumulators.hpp>
#include <boost/accumulators/statistics/stats.hpp>
#include <boost/accumulators/statistics/mean.hpp>
#include <boost/accumulators/statistics/moment.hpp>
#include <boost/chrono.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/qi_parse.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/fusion/include/vector.hpp>
#include <boost/spirit/include/karma.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/assign.hpp>
#include <iostream>
#include <string>
#include <vector>
#include <random>
using namespace boost::accumulators;
typedef boost::chrono::duration<long long, boost::micro> microseconds;
namespace qi = boost::spirit::qi;
namespace karma = boost::spirit::karma;
namespace ascii = qi::ascii;
namespace phx = boost::phoenix;
const std::map<const std::string, const std::string> mime_exts = {
{ ".htm", "text/html" },
{ ".html", "text/html" },
{ ".php", "text/html" },
{ ".css", "text/css" },
{ ".js", "application/javascript" },
{ ".json", "application/json" },
{ ".xml", "application/xml" },
{ ".swf", "application/x-shockwave-flash" },
{ ".flv", "video/x-flv" },
{ ".png", "image/png" },
{ ".jpe", "image/jpeg" },
{ ".jpeg", "image/jpeg" },
{ ".jpg", "image/jpeg" },
{ ".gif", "image/gif" },
{ ".bmp", "image/bmp" },
{ ".ico", "image/vnd.microsoft.icon" },
{ ".tif", "image/tiff" },
{ ".tiff", "image/tiff" },
{ ".svg", "image/svg+xml"},
{ ".svgz", "image/svg+xml"}
};
const char *mime_literals[] = {
"text/html",
"text/css",
"text/plain",
"application/javascript",
"application/json",
"application/xml",
"application/x-shockwave-flash",
"video/x-flv",
"image/png",
"image/jpeg",
"image/gif",
"image/bmp",
"image/vnd.microsoft.icon",
"image/tiff",
"image/svg+xml"
};
template <typename Iterator>
struct mimetype_matching_parser : qi::grammar<Iterator, unsigned int()> {
mimetype_matching_parser() : mimetype_matching_parser::base_type(m_start, "mimetype_matching_parser") {
m_mime_extensions.add
(".htm", 0)
(".html", 0)
(".php", 0)
(".css", 1)
(".txt", 2)
(".js", 3)
(".json", 4)
(".xml", 5)
(".swf", 6)
(".flv", 7)
(".png", 8)
(".jpe", 9)
(".jpeg", 9)
(".jpg", 9)
(".gif", 10)
(".bmp", 11)
(".ico", 12)
(".tiff", 13)
(".tif", 13)
(".svg", 14)
(".svgz", 14)
;
using qi::no_case;
m_start %= no_case[m_mime_extensions] >> qi::eoi;
}
qi::symbols<char, unsigned int> m_mime_extensions;
qi::rule<Iterator, unsigned int()> m_start;
};
std::string mime_extension(const std::string &n_path) {
// First locate the extension itself
const std::size_t last_dot = n_path.rfind(".");
if (last_dot == std::string::npos) {
return "application/text";
}
// and now pipe the extension into a qi symbols parser.
// I don't know if this is any faster than a more trivial algorithm::ends_with
// approach but I guess it won't be any slower
const mimetype_matching_parser<std::string::const_iterator> p;
unsigned int result;
std::string::const_iterator begin = n_path.begin() + last_dot;
const std::string::const_iterator end = n_path.end();
try {
if (qi::parse(begin, end, p, result) && (begin == end)) {
return mime_literals[result];
} else {
return "application/text";
}
} catch (const std::exception &) { // asio throws on invalid parse
return "application/text";
}
}
std::string mime_extension2(const std::string &n_path) {
using boost::algorithm::iequals;
auto const ext = [&n_path] {
auto const pos = n_path.rfind(".");
if (pos == std::string::npos)
return std::string{};
return n_path.substr(pos);
}();
// const std::size_t pos = n_path.rfind(".");
// if (pos == std::string::npos) {
// return std::string{};
// }
// const std::string ext = n_path.substr(pos);
if (iequals(ext, ".htm")) return "text/html";
if (iequals(ext, ".html")) return "text/html";
if (iequals(ext, ".php")) return "text/html";
if (iequals(ext, ".css")) return "text/css";
if (iequals(ext, ".txt")) return "text/plain";
if (iequals(ext, ".js")) return "application/javascript";
if (iequals(ext, ".json")) return "application/json";
if (iequals(ext, ".xml")) return "application/xml";
if (iequals(ext, ".swf")) return "application/x-shockwave-flash";
if (iequals(ext, ".flv")) return "video/x-flv";
if (iequals(ext, ".png")) return "image/png";
if (iequals(ext, ".jpe")) return "image/jpeg";
if (iequals(ext, ".jpeg")) return "image/jpeg";
if (iequals(ext, ".jpg")) return "image/jpeg";
if (iequals(ext, ".gif")) return "image/gif";
if (iequals(ext, ".bmp")) return "image/bmp";
if (iequals(ext, ".ico")) return "image/vnd.microsoft.icon";
if (iequals(ext, ".tiff")) return "image/tiff";
if (iequals(ext, ".tif")) return "image/tiff";
if (iequals(ext, ".svg")) return "image/svg+xml";
if (iequals(ext, ".svgz")) return "image/svg+xml";
return "application/text";
}
std::string mime_extension3(const std::string &n_path) {
using boost::algorithm::iequals;
auto ext = [&n_path] {
auto const pos = n_path.rfind(".");
if (pos == std::string::npos) {
return std::string{};
} else {
return n_path.substr(pos);
}
}();
boost::algorithm::to_lower(ext);
const std::map<const std::string, const std::string>::const_iterator i = mime_exts.find(ext);
if (i != mime_exts.cend()) {
return i->second;
} else {
return "application/text";
}
}
const std::string samples[] = {
"test.txt",
"test.html",
"longer/test.tiff",
"www.webSite.de/ico.ico",
"www.websIte.de/longEr/path/ico.bmp",
"www.TEST.com/longer/path/ico.svg",
"googlecom/shoRT/path/index.HTM",
"googlecom/bild.jpg",
"WWW.FLASH.COM/app.swf",
"WWW.FLASH.COM/BILD.GIF"
};
int test_qi_impl() {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<> dis(0, 10);
const std::string sample = samples[dis(gen)];
const std::string result = mime_extension(sample);
int ret = dis(gen);
for (const char &c : result) { ret += c; }
return ret;
}
int test_lambda_impl() {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<> dis(0, 10);
const std::string sample = samples[dis(gen)];
const std::string result = mime_extension2(sample);
int ret = dis(gen);
for (const char &c : result) { ret += c; }
return ret;
}
int test_map_impl() {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<> dis(0, 10);
const std::string sample = samples[dis(gen)];
const std::string result = mime_extension3(sample);
int ret = dis(gen);
for (const char &c : result) { ret += c; }
return ret;
}
int main(int argc, char **argv) {
const unsigned int loops = 100000;
accumulator_set<boost::chrono::high_resolution_clock::duration, features<tag::mean> > times_qi;
accumulator_set<boost::chrono::high_resolution_clock::duration, features<tag::mean> > times_lambda;
accumulator_set<boost::chrono::high_resolution_clock::duration, features<tag::mean> > times_map;
std::cout << "Measure execution times for " << loops << " lambda runs" << std::endl;
for (unsigned int i = 0; i < loops; i++) {
boost::chrono::high_resolution_clock::time_point start = boost::chrono::high_resolution_clock::now();
test_lambda_impl();
boost::chrono::high_resolution_clock::time_point end = boost::chrono::high_resolution_clock::now();
times_lambda(end - start);
}
std::cout << "Measure execution times for " << loops << " qi runs" << std::endl;
for (unsigned int i = 0; i < loops; i++) {
boost::chrono::high_resolution_clock::time_point start = boost::chrono::high_resolution_clock::now();
test_qi_impl();
boost::chrono::high_resolution_clock::time_point end = boost::chrono::high_resolution_clock::now();
times_qi(end - start);
}
std::cout << "Measure execution times for " << loops << " map runs" << std::endl;
for (unsigned int i = 0; i < loops; i++) {
boost::chrono::high_resolution_clock::time_point start = boost::chrono::high_resolution_clock::now();
test_map_impl();
boost::chrono::high_resolution_clock::time_point end = boost::chrono::high_resolution_clock::now();
times_map(end - start);
}
std::cout << "Lambda runs took " << mean(times_lambda) << std::endl;
std::cout << "Qi runs took " << mean(times_qi) << std::endl;
std::cout << "Map runs took " << mean(times_map) << std::endl;
return EXIT_SUCCESS;
}
Run Code Online (Sandbox Code Playgroud)
令我惊讶的是,lambda确实很重要(一点点).更令我惊讶的是,qi的实现速度要慢得多.
Measure execution times for 100000 lambda runs
Measure execution times for 100000 qi runs
Measure execution times for 100000 map runs
Lambda runs took 12443 nanoseconds
Qi runs took 15311 nanoseconds
Map runs took 10466 nanoseconds
Run Code Online (Sandbox Code Playgroud)
首先,我使用的是这样的符号
template <typename Iterator>
struct mimetype_matching_parser : qi::grammar<Iterator, std::string()> {
mimetype_matching_parser() : mimetype_matching_parser::base_type(m_start,
"mimetype_matching_parser") {
m_mime_extensions.add
(".htm", "text/html")
(".html", "text/html")
(".php", "text/html")
(".css", "text/css")
(".svg", "whatever...")
;
using qi::no_case;
m_start %= no_case[m_mime_extensions] >> qi::eoi;
}
qi::symbols<char, std::string> m_mime_extensions;
qi::rule<Iterator, std::string()> m_start;
};
Run Code Online (Sandbox Code Playgroud)
它直接将字符串作为属性返回.一位同事指出这是一个额外的std :: string副本,所以我改了它,所以它只返回一个静态char数组的索引:
const char *mime_literals[] = {
"text/html",
"text/css",
"text/plain",
// ... and so forth
};
template <typename Iterator>
struct mimetype_matching_parser : qi::grammar<Iterator, unsigned int()> {
mimetype_matching_parser() : mimetype_matching_parser::base_type(m_start, "mimetype_matching_parser")
{
m_mime_extensions.add
(".htm",0)
(".html",0)
(".php",0)
(".css",1)
(".svg",... etc.
;
using qi::no_case;
m_start %= no_case[m_mime_extensions] >> qi::eoi;
}
qi::symbols<char, unsigned int> m_mime_extensions;
qi::rule<Iterator, unsigned int()> m_start;
};
Run Code Online (Sandbox Code Playgroud)
这有点快,但不值得一提.
在我的笔记本电脑上,在发布模式下,我得到: - 野兽教程(Lambda)实现平均每次运行6200纳秒. - Qi实现平均值约为7100纳秒.
现在,这将是我的第一个问题:为什么?
"野兽"的实现让我觉得非常低效,因为它遍历所有的子串,每次调用iequals,而它能够缓存小写.
我认为qi肯定会对我添加到符号解析器的关键字进行二进制搜索.但看起来并不像.
所以我想出了我自己的,也很简单的实现,并使用带有缓存小写临时的静态映射(请参阅附加源中的impl3).
测试得出:
Sooo,我猜问题是为什么?
我qi::symbols不知何故误用了?它实际上是否进行了二进制搜索,但其他地方的性能却丢失了吗?
斯蒂芬 ¹
(我在Windows MSVC14 64位,增强1.66)
(¹这个问题是从Spirit General邮件列表中转述的,它在20180112T14:15CET发布; 在线档案似乎很遗憾)
在12-01-18 14:15,Stephan Menzel写道:
所以我提出了两种不同的实现方式.请找到附带的来源.
我看了看.首先是一些肤浅的观察:
你比较苹果和梨,因为Beast使用零拷贝的字符串视图,而Qi则没有.
此外,样本选择调用UB,因为uniform_int_distribution(0,10)它超出了样本数组的范围(应该是(0, 9)).
最后,map方法没有.txt扩展名的映射.
通过这些方法,我将测试程序简化/构建为以下内容:
在我的系统上打印以下内容:
Lambda runs took 2319 nanoseconds
Qi runs took 2841 nanoseconds
Map runs took 193 nanoseconds
Run Code Online (Sandbox Code Playgroud)
现在,最大的罪魁祸首(显然是?)你每次通过循环构建语法(编译规则).当然,没有必要.删除产量:
Lambda runs took 2676 nanoseconds
Qi runs took 98 nanoseconds
Map runs took 189 nanoseconds
Run Code Online (Sandbox Code Playgroud)
即使您在没有实际需要的情况下仍在复制字符串,这已经更快了.使用上面链接的答案的灵感,我可能会写它:
#include <boost/spirit/include/qi.hpp>
namespace qi_impl {
namespace qi = boost::spirit::qi;
struct mimetype_symbols_type : qi::symbols<char, char const*> {
mimetype_symbols_type() {
auto rev = [](string_view s) -> std::string { return { s.rbegin(), s.rend() }; };
this->add
(rev(".htm"), "text/html")
(rev(".html"), "text/html")
(rev(".php"), "text/html")
(rev(".css"), "text/css")
(rev(".txt"), "text/plain")
(rev(".js"), "application/javascript")
(rev(".json"), "application/json")
(rev(".xml"), "application/xml")
(rev(".swf"), "application/x-shockwave-flash")
(rev(".flv"), "video/x-flv")
(rev(".png"), "image/png")
(rev(".jpe"), "image/jpeg")
(rev(".jpeg"), "image/jpeg")
(rev(".jpg"), "image/jpeg")
(rev(".gif"), "image/gif")
(rev(".bmp"), "image/bmp")
(rev(".ico"), "image/vnd.microsoft.icon")
(rev(".tiff"), "image/tiff")
(rev(".tif"), "image/tiff")
(rev(".svg"), "image/svg+xml")
(rev(".svgz"), "image/svg+xml")
;
}
} static const mime_symbols;
char const* using_spirit(const string_view &n_path) {
char const* result = "application/text";
qi::parse(n_path.crbegin(), n_path.crend(), qi::no_case[mime_symbols], result);
return result;
}
}
Run Code Online (Sandbox Code Playgroud)
没有必要首先找到"最后一个点",不需要"检查匹配是否在最后",并且您可以直接从符号中获取值.您可以根据需要自由分配给a string_view或a std::string.
使用string_views(包括std::string_view和boost::string_view支持/所示)贯穿始终.
另请注意,这显示了在
map<>方法上使用的自定义比较器,只是为了证明确实知道地图键都是小写的,这确实有好处.(事实上,它不是因为它"缓存了小写",因为它只使用过一次!)
#include <boost/chrono.hpp>
#include <string>
#ifdef BOOST_STRING_VIEW
#include <boost/utility/string_view.hpp>
using string_view = boost::string_view;
#else
#include <string_view>
using string_view = std::string_view;
#endif
static auto constexpr npos = string_view::npos;
#include <boost/spirit/include/qi.hpp>
namespace qi_impl {
namespace qi = boost::spirit::qi;
struct mimetype_symbols_type : qi::symbols<char, char const*> {
mimetype_symbols_type() {
auto rev = [](string_view s) -> std::string { return { s.rbegin(), s.rend() }; };
this->add
(rev(".htm"), "text/html")
(rev(".html"), "text/html")
(rev(".php"), "text/html")
(rev(".css"), "text/css")
(rev(".txt"), "text/plain")
(rev(".js"), "application/javascript")
(rev(".json"), "application/json")
(rev(".xml"), "application/xml")
(rev(".swf"), "application/x-shockwave-flash")
(rev(".flv"), "video/x-flv")
(rev(".png"), "image/png")
(rev(".jpe"), "image/jpeg")
(rev(".jpeg"), "image/jpeg")
(rev(".jpg"), "image/jpeg")
(rev(".gif"), "image/gif")
(rev(".bmp"), "image/bmp")
(rev(".ico"), "image/vnd.microsoft.icon")
(rev(".tiff"), "image/tiff")
(rev(".tif"), "image/tiff")
(rev(".svg"), "image/svg+xml")
(rev(".svgz"), "image/svg+xml")
;
}
} static const mime_symbols;
char const* using_spirit(const string_view &n_path) {
char const* result = "application/text";
qi::parse(n_path.crbegin(), n_path.crend(), qi::no_case[mime_symbols], result);
return result;
}
}
#include <boost/algorithm/string.hpp>
namespace impl {
string_view using_iequals(const string_view &n_path) {
using boost::algorithm::iequals;
auto const ext = [&n_path] {
auto pos = n_path.rfind(".");
return pos != npos? n_path.substr(pos) : string_view {};
}();
if (iequals(ext, ".htm")) return "text/html";
if (iequals(ext, ".html")) return "text/html";
if (iequals(ext, ".php")) return "text/html";
if (iequals(ext, ".css")) return "text/css";
if (iequals(ext, ".txt")) return "text/plain";
if (iequals(ext, ".js")) return "application/javascript";
if (iequals(ext, ".json")) return "application/json";
if (iequals(ext, ".xml")) return "application/xml";
if (iequals(ext, ".swf")) return "application/x-shockwave-flash";
if (iequals(ext, ".flv")) return "video/x-flv";
if (iequals(ext, ".png")) return "image/png";
if (iequals(ext, ".jpe")) return "image/jpeg";
if (iequals(ext, ".jpeg")) return "image/jpeg";
if (iequals(ext, ".jpg")) return "image/jpeg";
if (iequals(ext, ".gif")) return "image/gif";
if (iequals(ext, ".bmp")) return "image/bmp";
if (iequals(ext, ".ico")) return "image/vnd.microsoft.icon";
if (iequals(ext, ".tiff")) return "image/tiff";
if (iequals(ext, ".tif")) return "image/tiff";
if (iequals(ext, ".svg")) return "image/svg+xml";
if (iequals(ext, ".svgz")) return "image/svg+xml";
return "application/text";
}
}
#include <boost/algorithm/string.hpp>
#include <map>
namespace impl {
struct CiCmp {
template <typename R1, typename R2>
bool operator()(R1 const& a, R2 const& b) const {
return boost::algorithm::ilexicographical_compare(a, b);
}
};
static const std::map<string_view, string_view, CiCmp> s_mime_exts_map {
{ ".txt", "text/plain" },
{ ".htm", "text/html" },
{ ".html", "text/html" },
{ ".php", "text/html" },
{ ".css", "text/css" },
{ ".js", "application/javascript" },
{ ".json", "application/json" },
{ ".xml", "application/xml" },
{ ".swf", "application/x-shockwave-flash" },
{ ".flv", "video/x-flv" },
{ ".png", "image/png" },
{ ".jpe", "image/jpeg" },
{ ".jpeg", "image/jpeg" },
{ ".jpg", "image/jpeg" },
{ ".gif", "image/gif" },
{ ".bmp", "image/bmp" },
{ ".ico", "image/vnd.microsoft.icon" },
{ ".tif", "image/tiff" },
{ ".tiff", "image/tiff" },
{ ".svg", "image/svg+xml"},
{ ".svgz", "image/svg+xml"},
};
string_view using_map(const string_view& n_path) {
auto const ext = [](string_view n_path) {
auto pos = n_path.rfind(".");
return pos != npos? n_path.substr(pos) : string_view {};
};
auto i = s_mime_exts_map.find(ext(n_path));
if (i != s_mime_exts_map.cend()) {
return i->second;
} else {
return "application/text";
}
}
}
#include <random>
namespace samples {
static string_view const s_samples[] = {
"test.txt",
"test.html",
"longer/test.tiff",
"www.webSite.de/ico.ico",
"www.websIte.de/longEr/path/ico.bmp",
"www.TEST.com/longer/path/ico.svg",
"googlecom/shoRT/path/index.HTM",
"googlecom/bild.jpg",
"WWW.FLASH.COM/app.swf",
"WWW.FLASH.COM/BILD.GIF"
};
std::mt19937 s_random_generator(std::random_device{}());
std::uniform_int_distribution<> s_dis(0, boost::size(s_samples) - 1);
string_view random_sample() {
return s_samples[s_dis(s_random_generator)];
}
}
#include <boost/functional/hash.hpp>
#include <iostream>
template <typename F>
int generic_test(F f) {
auto sample = samples::random_sample();
string_view result = f(sample);
//std::cout << "DEBUG " << sample << " -> " << result << "\n";
return boost::hash_range(result.begin(), result.end());
}
#include <boost/serialization/array_wrapper.hpp> // missing include in boost version on coliru
#include <boost/accumulators/accumulators.hpp>
#include <boost/accumulators/statistics.hpp>
template <typename F>
auto benchmark(F f) {
using C = boost::chrono::high_resolution_clock;
using duration = C::duration;
const unsigned int loops = 100000;
namespace ba = boost::accumulators;
ba::accumulator_set<duration, ba::features<ba::tag::mean>> times;
for (unsigned int i = 0; i < loops; i++) {
auto start = C::now();
generic_test(f);
times(C::now() - start);
}
return ba::mean(times);
}
int main() {
std::cout << std::unitbuf;
std::cout << "Lambda runs took " << benchmark(impl::using_iequals) << std::endl;
std::cout << "Qi runs took " << benchmark(qi_impl::using_spirit) << std::endl;
std::cout << "Map runs took " << benchmark(impl::using_map) << std::endl;
}
Run Code Online (Sandbox Code Playgroud)
打印
Lambda runs took 2470 nanoseconds
Qi runs took 119 nanoseconds
Map runs took 2239 nanoseconds // see Note above
Run Code Online (Sandbox Code Playgroud)
| 归档时间: |
|
| 查看次数: |
176 次 |
| 最近记录: |