/*************************** @Author: Chunel @Contact: chunel@foxmail.com @File: E04-MapReduce.cpp @Time: 2024/1/26 22:58 @Desc: ***************************/ #include #include #include "CGraph.h" using namespace CGraph; static const int MAP_SIZE = 4; static const int NUMS_PER_MAP = 1000000; static const int MAX_NUM = 10; static const char* PARAM_KEY = "param-key"; struct NumsGParam : public GParam { std::vector random_nums_ {}; // 随机生成的数字信息 int map_results_[MAP_SIZE][MAX_NUM] = {}; // 用于记录 map 处理结果的信息 }; class GenerateGNode : public GNode { public: CStatus init() override { auto status = CGRAPH_CREATE_GPARAM(NumsGParam, PARAM_KEY); return status; } CStatus run() override { std::mt19937 generator; auto param = CGRAPH_GET_GPARAM_WITH_NO_EMPTY(NumsGParam, PARAM_KEY); std::uniform_int_distribution distribution(0, MAX_NUM - 1); for (int i = 0; i < MAP_SIZE * NUMS_PER_MAP; i++) { int num = distribution(generator); param->random_nums_.push_back(num); } return CStatus(); } }; template class MapGNode : public GNode { public: CStatus run() override { auto param = CGRAPH_GET_GPARAM_WITH_NO_EMPTY(NumsGParam, PARAM_KEY); for (int i = 0; i < NUMS_PER_MAP; i++) { int num = param->random_nums_[MAP_TAG * NUMS_PER_MAP + i]; param->map_results_[MAP_TAG][num]++; // 实际上是线程安全的 } return CStatus(); } }; class ReduceGNode : public GNode { public: CStatus run() override { int total = 0; auto param = CGRAPH_GET_GPARAM_WITH_NO_EMPTY(NumsGParam, PARAM_KEY); for (int j = 0; j < MAX_NUM; j++) { int num = 0; for (int i = 0; i < MAP_SIZE; i++) { num += param->map_results_[i][j]; } total += num; printf(" num [%d] appear [%d] times\n", j, num); } printf("total num is [%d]. \n", total); return CStatus(); } }; void example_map_reduce() { auto pipeline = GPipelineFactory::create(); GElementPtr a, b0, b1, b2, b3, c = nullptr; CStatus status; status += pipeline->registerGElement(&a, {}); status += pipeline->registerGElement>(&b0, {a}); status += pipeline->registerGElement>(&b1, {a}); status += pipeline->registerGElement>(&b2, {a}); status += pipeline->registerGElement>(&b3, {a}); status += pipeline->registerGElement(&c, {b0, b1, b2, b3}); status += pipeline->process(); if (status.isErr()) { CGRAPH_ECHO("example_map_reduce error info is : %s", status.getInfo().c_str()); } GPipelineFactory::clear(); } int main() { example_map_reduce(); return 0; }