diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..a5ab743 Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index e660fd9..469a259 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ bin/ +*.swp +*.so +*.o +libsrvcli/ +benchmarks/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..f1aa480 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "benchmarks/sorting-benchmarks"] + path = benchmarks/sorting-benchmarks + url = https://github.com/dopecoder/sorting-benchmark.git +[submodule "benchmarks/ram-bench"] + path = benchmarks/ram-bench + url = https://github.com/dopecoder/ram_bench.git diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e03956d --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,70 @@ +{ + "files.associations": { + "iostream": "cpp", + "ostream": "cpp", + "*.tcc": "cpp", + "vector": "cpp", + "__hash_table": "cpp", + "unordered_map": "cpp", + "__bit_reference": "cpp", + "__config": "cpp", + "__debug": "cpp", + "__errc": "cpp", + "__functional_base": "cpp", + "__locale": "cpp", + "__mutex_base": "cpp", + "__node_handle": "cpp", + "__nullptr": "cpp", + "__split_buffer": "cpp", + "__string": "cpp", + "__threading_support": "cpp", + "__tuple": "cpp", + "algorithm": "cpp", + "array": "cpp", + "atomic": "cpp", + "bit": "cpp", + "bitset": "cpp", + "cctype": "cpp", + "chrono": "cpp", + "cmath": "cpp", + "complex": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "exception": "cpp", + "functional": "cpp", + "initializer_list": "cpp", + "ios": "cpp", + "iosfwd": "cpp", + "istream": "cpp", + "iterator": "cpp", + "limits": "cpp", + "locale": "cpp", + "memory": "cpp", + "mutex": "cpp", + "new": "cpp", + "optional": "cpp", + "ratio": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "typeinfo": "cpp", + "utility": "cpp", + "deque": "cpp", + "fstream": "cpp", + "iomanip": "cpp", + "numeric": "cpp", + "stack": "cpp" + } +} \ No newline at end of file diff --git a/Makefile b/Makefile index 76e3294..1440154 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,28 @@ -CC=g++ -CFLAGS= +CC:=g++ +CFLAGS:=-fPIC -c +LDFLAGS:=-Iinclude -Ilibsrvcli/include -lpthread -all: client server - -client: bin_dir - $(CC) client/*.cpp -o bin/client +ifdef $(DEBUG) +CFLAGS += -DDEBUG_PRINTS=$(DEBUG) +endif -server: bin_dir - $(CC) server/*.cpp -o bin/server +all: libsrvcli/libsrvcli.so rmp_client.o rmp_server.o + $(CC) -shared -o rmp.so libsrvcli/libsrvcli.so rmp_server.o rmp_client.o $(LDFLAGS) -bin_dir: - mkdir -p bin +libsrvcli/libsrvcli.so: + $(MAKE) -C libsrvcli/ -clean: - rm -r bin - +rmp_server.o: server/rmp_server.cpp + $(CC) $(CFLAGS) server/rmp_server.cpp -o rmp_server.o $(LDFLAGS) + +rmp_client.o: client/rmp_client.cpp uffdman.o + $(CC) $(CFLAGS) client/rmp_client.cpp -o rmp_client.o $(LDFLAGS) -luffdman.o + +uffdman.o: uffdman/uffdman.cpp + $(CC) $(CFLAGS) uffdman/uffdman.cpp -o uffdman.o $(LDFLAGS) + +clean: + $(MAKE) -C libsrvcli/ clean + rm -f rmp.so rmp_server.o rmp_client.o uffdman.o + +.PHONY: libsrvcli/libsrvcli.so diff --git a/README.md b/README.md index 8510931..6aae921 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,313 @@ -# rmpool: Remote Memory Pool -## An extensible mechanism for pooling memory among remote machines + + +# RMP: Remote Memory Pool +### A Research project under [Prof. Kyle Hale](https://www.halek.co/): +* [Nithin Rao](https://www.halek.co/authors/nithin-rao/) +* [Nanda Velugoti](https://www.halek.co/authors/nanda-velugoti/) + +## To Run the test program, + +``` +cd server +make +./bin/server 127.0.0.1 6768 +``` + +``` +cd client +make +./bin/client 127.0.0.1 6768 +``` + +## Abstract + +Memory management is one of the key components ofany software system +environment. For most of the cases local memory (localphysical RAM) to allocate, read, write +and free memory regions in a program. In these casesthe amount of memory available to it is +limited for a program. Remote Memory Pool (RMP) isdesigned to address this memory +limitation, by providing a way for programs to allocatememory remotely. We do this by +implementing RMP as a user-level library that allowsusers to set up a server program that +servers the remote memory and a client program canuse that remote memory from the server. +Not only that RMP does this exposing a truly minimalAPI, just three to be exact, for the client +program to allocate remote memory. Also the readsand writes to this remote memory does not +involve invocation of any external API or methods.Reading and writing is done just like a +variable access in C. To achieve this we use somelinux kernel mechanisms such as userfaultfd +to resolve reads and writes under the hood. This meansthat users can start using RMP without +any major changes to their existing program code. + +## Introduction + +In the evolution of distributed systems, managingmemory gets even more difficult. Many +of the implementations of remote memory expose APIsfor reading and writing data to memory +remotely, which makes it very hard for the developersto integrate such implementations into +their existing programs to use remote memory. Evenfor developers to write new programs with +these remote memory implementations need to use APIslike read/write to do memory +operations. + +Remote Memory Pool (RMP) is implemented as a userspace library that exposes basic +initialization, allocation and freeing APIs for usersto manage remote memory. All the reads and +writes to this is remote memory is done as usual meaningusing variables. + +In normal use cases of memory allocation and access,the memory is allocated in the +local physical RAM of the machine. With our implementationof RMP (Remote Memory Pool) +users can allocate memory, with page granularity,on a remote server’s physical RAM and +access (Read/Write) in real-time on demand. The objectivehere is to provide the user with a +minimal API that looks like a generic memory allocationAPI, such as malloc, while taking care +of sending and receiving the data back and forth withoutthe user having to change his code +significantly. + +What exactly are we trying to accomplish? Below arethe examples illustrating how memory is +allocated, accessed and freed locally (normal) andremotely (RMP). + + +``` +Accessing Local Memory +``` +``` +Accessing Remote Memory (RMP) +``` +## Userfaultfd + +The userfaultfd mechanism is designed to allow a threadin a multithreaded program to +perform user-space paging for the other threads inthe process. When a page fault occurs for +one of the regions registered to the userfaultfd object,the faulting thread is put to sleep and an +event is generated that can be read via the userfaultfdfile descriptor. The fault-handling thread +reads events from this file descriptor and servicesthem using operations described in +ioctl_userfaultfd(2). When servicing the page faultevents, the fault-handling thread can trigger +a wake-up for the sleeping thread. + +It is possible for the faulting threads and the fault-handlingthreads to run in the context +of processes. In this case, these threads may belongto differ‐ ent programs, and the program +that executes the faulting threads will not necessarilycooperate with those that handle the page + + +faults. In such non-cooperative mode, the process that monitors userfaultfd and handles page +faults needs to be aware of the changes in the memorylayout of the faulting process to avoid +mem‐ ory corruption. + +After the userfaultfd object is created with userfaultfd(),the application must enable it +using the UFFDIO_API ioctl(2) operation. This operationallows a handshake between the +kernel and user space to determine the API versionand supported features. This operation +must be performed before any of the other ioctl(2)operations de‐ scribed below (or those +operations fail with the EINVAL error). + +After a successful UFFDIO_API operation, the applicationthen registers memory +address ranges using UFFDIO_REGISTER ioctl(2) operation. After successful completion of a +UFFDIO_REGISTER operation, a page fault occurringin the requested memory range, and +satisfying the defined at the registration time, willbe forwarded by the kernel to the user-space +application. The application can then use the UFFDIO_COPYor UFFDIO_ZEROPAGE ioctl( +operations to resolve the page fault. + + +## Problems with userfaultfd + +**There are two basic problems:** + +1. A pagefault (userfault) won't occur for subsequentreads/writes once it is + resolved/handled. +2. In a pagefault due to a write, we won't get the data(part of faulting instruction) that is + being written in the userfault handler. + +## Our Solution: Off-by-one invalidation + +**Off-by-one Invalidation:** If a pagefault occurs, thenthe faulting address and operation +(read/write) is saved, so that it can be invalidatedwhen the next pagefault occurs. + +### Constraints: + +1. At Most two pages of memory is required. +2. The memory of the entity which is resolving pageswill be behind by one memory + address in terms of consistency. Since the name Off-by-one. + +## RMP Components + +RMP as library has a + +**Server Component** +● Exposes user API to setup RMP server: rmp_server_init() +● Serves pages to clients by sending a page on readand receiving a page on write. +**Client Component** +● Registers with userfaultfd manager to listen to pagefaults +● Exposes user API: rmp_init(), rmp_alloc(n_pages),rmp_free(addr) +● Communicates with the server by sending a page onwrite and receiving a page on read. +**Userfaultfd Manager** +● Handles user page faults on the registered region. +● A fault handler thread gets an userfault event withfault address and operation type +(R/W) + + +## Under the hood of RMP + +**Process of allocation, access and freeing of remotememory:** + +1. Client requests to allocate n pages on a remote server. +2. Server allocates the n pages in memory and returnsa mapped handle. +3. Client stores the handle by mapping it to the localaddress. +4. Client then registers the local region with UFFD Manager +5. UFFD Manager then registers the region as a UFFD regionand starts a fault handler + thread for that region. +6. A read/write to this region will cause a page faultthat will be caught by the appropriate + fault handler thread, which then is forwarded to theRMP Client to resolve the fault. +7. This read/write will be resolved by requesting theserver by sending the appropriate + handle that was stored in step 3 and the calculatedoffset of the page within the region. + The offset is calculated by, + Offset =(Faulting Address - Region Start Address) / Page Size +8. After repeating step 7 for all the accesses to allthe remote memory regions, the client + then can send a free request to Server, which serverfrees the region in it’s memory after + which the client does the same. + + +## Using RMP: Server/Client API and Example + +## Programs + +**Server API:** +● void rmp_server_init(server_conn_config); +**Client API:** +● void rmp_init(server_conn_config); +● void* rmp_alloc(long n_pages); +● void rmp_free(void* addr); + +``` +Client.c Server.c +``` +## Testing and Evaluation + +### Testing Setup: + +``` +Server 4 Core, 8GB RAM, Gigabit Ethernet +``` +``` +Client 4 Core, 1GB RAM, Gigabit Ethernet +``` + +### Test Suites: + +``` +● Program to allocate 1000, 10000, 100000 sized arraysand fill them up with numbers +and calculate the sum. + ○ Sequential Read Write Execution + ○ Random Read Write Execution +● Insertion Sort Algorithm +``` +### Results + +**1. Sequential Access** + +``` +Operation Type Mode Time (seconds) +``` +``` +Sequential Read Local 2.59893e- +``` +``` +Sequential Read Remote 2.3117e- +``` +``` +Sequential Write Local 3.22393e- +``` +``` +Sequential Write Remote 6.4034e- +``` +**Observation:** Sequential remote accesses are slowerin the order of 10^4 microseconds + +**2. Random Access** + +``` +Operation Type Mode Time (seconds) +``` +``` +Random Read Local 0.0774958e- +``` +``` +Random Read Remote 0. +``` +``` +Random Write Local 0.0857879e- +``` +``` +Random Write Remote 0. +``` +**Observation:** Random remote accesses are slower inthe order of 10^9 microseconds + +3. **Insertion Sort (** 100000 Elements) + +``` +Local Execution Time 10.7446 Seconds +``` +``` +Remote Execution Time 199.1882 Seconds +``` + +**Observation:** Remote accesses are slower in the orderof 20x compared to local accesses + +### Graphs + +**Graphs for Insertion Sort with RMP:** 1000 to 1M elements + +``` +Linear view Logarithmic view +``` +## RMP Advantages and Disadvantages + +``` +Advantages Disadvantages +``` +``` +Clients can access more memory than it’s +own local memory (given that remote +machines can satisfy the client’s +requirement). +``` +``` +Not as fast as local memory. Speed of the +memory accesses is dependent on the +network communication type (RDMA, TCP/IP, +etc). +``` +``` +Minimal development or integration or porting +effort required. +``` +``` +Scaling and load balancing is not taken care +of in the current implementation. +``` +``` +Extensible, meaning that this implementation +can be extended to support many more +features. (see next section). +``` +``` +Currently every read/write will result in a +network communication (which is costly). This +can be solved by implementing a client side +cache. +``` +``` +This can be extended to implement +distributed remote memory. +``` +``` +Handling consistency and recovery in a +scenario with unexpected crashes is really +difficult as a lot state information and logging +is required +``` + +## Possible Improvements and Extensions + +``` +● Use upcoming linux kernel patches for userfaultfdthat resolves the problems that we +faced earlier instead of using the Off-by-one invalidationsolution proposed before. +● Use RDMA instead of TCP/IP (expected performance improvements) + ○ Hardware (RoCE or Infiniband) + ○ Software (SoftRoCE) +● Extend it to work with NDP (Near Data Processing)units. +● Support multiple servers and clients (distributedremote memory) +● A distributed memory with +● Discovery mechanism for available RMP servers +● Nodes in the network act as both servers and clients. +``` diff --git a/benchmarks/build.sh b/benchmarks/build.sh new file mode 100755 index 0000000..b5f27ee --- /dev/null +++ b/benchmarks/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +g++ throughput_benchmark.cpp -o tb +# g++ latency-benchmark.cpp -o lb \ No newline at end of file diff --git a/benchmarks/buildnbench-local.sh b/benchmarks/buildnbench-local.sh new file mode 100755 index 0000000..0b32c96 --- /dev/null +++ b/benchmarks/buildnbench-local.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# create a output directory +mkdir -p data +mkdir -p data/sorting-benchmarks +mkdir -p data/ram-bench +mkdir -p data/sysbench + + +# SORTING-BENCHMARK + +printf "Started SORTING-BENCHMARK" + +cd sorting-benchmarks + +cmake CMakeLists.txt +make + +./runtime_local.sh + +cd .. + +printf "Finished SORTING-BENCHMARK\n" + + +# RAM-BENCH + +printf "Running RAM-BENCH" + +cd ram-bench + +./build_and_run_local.sh + +cd .. + +printf "Finished RAM-BENCH\n" + +# SYSBENCH + +printf "Started SYSBENCH" + +sysbench --test=memory --memory-block-size=64K --memory-total-size=100G --num-threads=1 run > ./data/sysbench/sysbench-64K-local + +sysbench --test=memory --memory-block-size=1M --memory-total-size=100G --num-threads=1 run > ./data/sysbench/sysbench-1M-local + +sysbench --test=memory --memory-block-size=4M --memory-total-size=100G --num-threads=1 run > ./data/sysbench/sysbench-4M-local + +printf "Finished SYSBENCH\n" + +## Use perf to display the number of pagefaults for an example program such as malloc + +# MEMORY TEST + +# memory_test 0 21 > mem-test-local +# LD_PRELOAD=../preloadlib/bin/preloadlib.so memory_test 0 21 > mem-test-remote + +# MAGICK CONVERT + +# LD_PRELOAD=../preloadlib/bin/preloadlib.so time convert img.jpg img.png > convert-1-remote 2>&1 +# time convert img.jpg img.png > convert-1-local 2>&1 + +# LD_PRELOAD=../preloadlib/bin/preloadlib.so time convert input.png -colorspace RGB +sigmoidal-contrast 11.6933 \ +# -define filter:filter=Sinc -define filter:window=Jinc -define filter:lobes=3 \ +# -resize 400% -sigmoidal-contrast 11.6933 -colorspace sRGB output.png > convert-2-remote 2>&1 + +# time convert input.png -colorspace RGB +sigmoidal-contrast 11.6933 \ +# -define filter:filter=Sinc -define filter:window=Jinc -define filter:lobes=3 \ +# -resize 400% -sigmoidal-contrast 11.6933 -colorspace sRGB output.png > convert-2-local 2>&1 diff --git a/benchmarks/buildnbench-remote.sh b/benchmarks/buildnbench-remote.sh new file mode 100755 index 0000000..fdcdfe2 --- /dev/null +++ b/benchmarks/buildnbench-remote.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# create a output directory +mkdir -p data +mkdir -p data/sorting-benchmarks +mkdir -p data/ram-bench +mkdir -p data/sysbench + + +# # SORTING-BENCHMARK + +# printf "Started SORTING-BENCHMARK" + +# cd sorting-benchmarks + +# cmake CMakeLists.txt +# make + +# ./runtime_remote.sh + +# cd .. + +# printf "Finished SORTING-BENCHMARK\n" + + +# # RAM-BENCH + +# printf "Running RAM-BENCH" + +# cd ram-bench + +# ./build_and_run_remote.sh + +# cd .. + +# printf "Finished RAM-BENCH\n" + +# SYSBENCH + +printf "Started SYSBENCH" + +LD_PRELOAD=../preloadlib/bin/preloadlib.so sysbench --test=memory --memory-block-size=64K --memory-total-size=100G --num-threads=1 run > ./data/sysbench/sysbench-64K-remote + +LD_PRELOAD=../preloadlib/bin/preloadlib.so sysbench --test=memory --memory-block-size=1M --memory-total-size=100G --num-threads=1 run > ./data/sysbench/sysbench-1M-remote + +LD_PRELOAD=../preloadlib/bin/preloadlib.so sysbench --test=memory --memory-block-size=4M --memory-total-size=100G --num-threads=1 run > ./data/sysbench/sysbench-4M-remote + +printf "Finished SYSBENCH\n" + +## Use perf to display the number of pagefaults for an example program such as malloc + +# MEMORY TEST + +# memory_test 0 21 > mem-test-local +# LD_PRELOAD=../preloadlib/bin/preloadlib.so memory_test 0 21 > mem-test-remote + +# MAGICK CONVERT + +# LD_PRELOAD=../preloadlib/bin/preloadlib.so time convert img.jpg img.png > convert-1-remote 2>&1 +# time convert img.jpg img.png > convert-1-local 2>&1 + +# LD_PRELOAD=../preloadlib/bin/preloadlib.so time convert input.png -colorspace RGB +sigmoidal-contrast 11.6933 \ +# -define filter:filter=Sinc -define filter:window=Jinc -define filter:lobes=3 \ +# -resize 400% -sigmoidal-contrast 11.6933 -colorspace sRGB output.png > convert-2-remote 2>&1 + +# time convert input.png -colorspace RGB +sigmoidal-contrast 11.6933 \ +# -define filter:filter=Sinc -define filter:window=Jinc -define filter:lobes=3 \ +# -resize 400% -sigmoidal-contrast 11.6933 -colorspace sRGB output.png > convert-2-local 2>&1 diff --git a/benchmarks/install-deps.sh b/benchmarks/install-deps.sh new file mode 100755 index 0000000..cd28469 --- /dev/null +++ b/benchmarks/install-deps.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo apt install gnuplot +sudo apt install clang +sudo apt install cmake +sudo apt install libc++-dev diff --git a/benchmarks/latency-benchmark.cpp b/benchmarks/latency-benchmark.cpp new file mode 100644 index 0000000..6e39d9b --- /dev/null +++ b/benchmarks/latency-benchmark.cpp @@ -0,0 +1,291 @@ +#include +#include +#include +#include +#include +#include +#include + +// void print(const char *str) +// { +// write(STDOUT_FILENO, str, strlen(str)); +// } + +void isIntSumValid(int num, int sum) +{ + if (sum == ((num * (num - 1)) / 2)) + { + std::cout << "SUM is valid" << std::endl; + } + else + { + std::cout << "SUM is invalid" << std::endl; + } +} + +void isDoubleSumValid(double num, double sum) +{ + if (sum == ((num * (num - 1)) / 2)) + { + std::cout << "SUM is valid" << std::endl; + } + else + { + std::cout << "SUM is invalid" << std::endl; + } +} + +// Function to return the next random number +int getNum(std::vector &v) +{ + + // Size of the vector + int n = v.size(); + + // Generate a random number + srand(time(NULL)); + + // Make sure the number is within + // the index range + int index = rand() % n; + + // Get random number from the vector + int num = v[index]; + + // Remove the number from the vector + std::swap(v[index], v[n - 1]); + v.pop_back(); + + // Return the removed number + return num; +} + +// Function to generate n non-repeating random numbers +std::vector getIndexArray(int n) +{ + std::vector v(n); + + // Fill the vector with the values + // 1, 2, 3, ..., n + for (int i = 0; i < n; i++) + v[i] = i + 1; + + return v; +} + +void bench_for_int(int num) +{ + int n = num * sizeof(int); + int sum = 0; + + int *p = (int *)malloc(n); + std::chrono::duration total_latency + std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < (n / sizeof(int)); i++) + { + printf("%d\n", i); + std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); + p[i] = i; + } + std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < (n / sizeof(int)); i++) + { + sum += p[i]; + } + std::chrono::high_resolution_clock::time_point t3 = std::chrono::high_resolution_clock::now(); + + std::chrono::duration total_time = std::chrono::duration_cast>(t3 - t1); + std::chrono::duration read_time = std::chrono::duration_cast>(t3 - t2); + std::chrono::duration write_time = std::chrono::duration_cast>(t2 - t1); + + std::cout << "--------------INT SEQ " << num << " ---------------" << std::endl; + + // write time + std::cout << "Write Time " << write_time.count() << " seconds."; + // read time + std::cout << "Read Time " << read_time.count() << " seconds."; + // total time + std::cout << "Total Time " << total_time.count() << " seconds."; + // Avg read time / element + std::cout << "Avg Read Time " << read_time.count() / num << " seconds."; + // Avg write time / element + std::cout << "Avg Write Time " << write_time.count() / num << " seconds."; + // Avg total time / element + std::cout << "Avg Total Time " << total_time.count() / num << " seconds."; + + // Sum + std::cout << "Sum " << sum << std::endl; + + isIntSumValid(num, sum); +} + +void bench_for_double(int num) +{ + int n = num * sizeof(double); + double sum = 0; + double j = 0; + + double *p = (double *)malloc(n); + std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < (n / sizeof(double)); i++) + { + printf("%d\n", i); + p[i] = j; + j++; + } + std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < (n / sizeof(double)); i++) + { + sum += p[i]; + } + std::chrono::high_resolution_clock::time_point t3 = std::chrono::high_resolution_clock::now(); + + std::chrono::duration total_time = std::chrono::duration_cast>(t3 - t1); + std::chrono::duration read_time = std::chrono::duration_cast>(t3 - t2); + std::chrono::duration write_time = std::chrono::duration_cast>(t2 - t1); + + std::cout << "--------------DOUBLE SEQ " << num << " ---------------" << std::endl; + + // write time + std::cout << "Write Time " << write_time.count() << " seconds." << std::endl; + // read time + std::cout << "Read Time " << read_time.count() << " seconds." << std::endl; + // total time + std::cout << "Total Time " << total_time.count() << " seconds." << std::endl; + // Avg read time / element + std::cout << "Avg Read Time " << read_time.count() / num << " seconds." << std::endl; + // Avg write time / element + std::cout << "Avg Write Time " << write_time.count() / num << " seconds." << std::endl; + // Avg total time / element + std::cout << "Avg Total Time " << total_time.count() / num << " seconds." << std::endl; + + // Sum + std::cout << "Sum " << sum << std::endl; + isDoubleSumValid((double)num, sum); +} + +void bench_for_int_random(int num) +{ + int n = num * sizeof(int); + int sum = 0; + int i = 0; + + int *p = (int *)malloc(n); + std::vector idx1 = getIndexArray(num); + std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now(); + + while (idx1.size()) + { + i = getNum(idx1); + p[i] = i; + } + std::vector idx2 = getIndexArray(num); + + std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); + while (idx2.size()) + { + i = getNum(idx2); + sum += p[i]; + } + std::chrono::high_resolution_clock::time_point t3 = std::chrono::high_resolution_clock::now(); + + std::chrono::duration total_time = std::chrono::duration_cast>(t3 - t1); + std::chrono::duration read_time = std::chrono::duration_cast>(t3 - t2); + std::chrono::duration write_time = std::chrono::duration_cast>(t2 - t1); + + std::cout << "--------------INT RAND " << num << " ---------------" << std::endl; + + // write time + std::cout << "Write Time " << write_time.count() << " seconds." << std::endl; + // read time + std::cout << "Read Time " << read_time.count() << " seconds." << std::endl; + // total time + std::cout << "Total Time " << total_time.count() << " seconds." << std::endl; + // Avg read time / element + std::cout << "Avg Read Time " << read_time.count() / num << " seconds." << std::endl; + // Avg write time / element + std::cout << "Avg Write Time " << write_time.count() / num << " seconds." << std::endl; + // Avg total time / element + std::cout << "Avg Total Time " << total_time.count() / num << " seconds." << std::endl; + + // Sum + std::cout << "Sum " << sum << std::endl; + isIntSumValid(num, sum); +} + +void bench_for_double_random(int num) +{ + int n = num * sizeof(double); + int sum = 0; + int i = 0; + double j = 0; + + double *p = (double *)malloc(n); + std::vector idx1 = getIndexArray(num); + std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now(); + + while (idx1.size()) + { + i = getNum(idx1); + p[i] = i; + } + std::vector idx2 = getIndexArray(num); + + std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); + while (idx2.size()) + { + i = getNum(idx2); + sum += p[i]; + } + std::chrono::high_resolution_clock::time_point t3 = std::chrono::high_resolution_clock::now(); + + std::chrono::duration total_time = std::chrono::duration_cast>(t3 - t1); + std::chrono::duration read_time = std::chrono::duration_cast>(t3 - t2); + std::chrono::duration write_time = std::chrono::duration_cast>(t2 - t1); + + std::cout << "--------------DOUBLE RAND " << num << " ---------------" << std::endl; + + // write time + std::cout << "Write Time " << write_time.count() << " seconds."; + // read time + std::cout << "Read Time " << read_time.count() << " seconds."; + // total time + std::cout << "Total Time " << total_time.count() << " seconds."; + // Avg read time / element + std::cout << "Avg Read Time " << read_time.count() / num << " seconds."; + // Avg write time / element + std::cout << "Avg Write Time " << write_time.count() / num << " seconds."; + // Avg total time / element + std::cout << "Avg Total Time " << total_time.count() / num << " seconds."; + + // Sum + std::cout << "Sum " << sum << std::endl; + isDoubleSumValid((double)num, sum); +} + +int main(int argc, char *argv[]) +{ + bench_for_int(1024); + bench_for_int(10240); + bench_for_int(102400); + bench_for_int(1024000); + + bench_for_double(1024); + bench_for_double(10240); + bench_for_double(102400); + bench_for_double(1024000); + + bench_for_int_random(1024); + bench_for_int_random(10240); + bench_for_int_random(102400); + bench_for_int_random(1024000); + + bench_for_double_random(1024); + bench_for_double_random(10240); + bench_for_double_random(102400); + bench_for_double_random(1024000); +} \ No newline at end of file diff --git a/benchmarks/ram-bench b/benchmarks/ram-bench new file mode 160000 index 0000000..250f65d --- /dev/null +++ b/benchmarks/ram-bench @@ -0,0 +1 @@ +Subproject commit 250f65db877b4eb46419f727801d7ca0f244ea3b diff --git a/benchmarks/sorting-benchmarks b/benchmarks/sorting-benchmarks new file mode 160000 index 0000000..caadc4a --- /dev/null +++ b/benchmarks/sorting-benchmarks @@ -0,0 +1 @@ +Subproject commit caadc4a87d42c3278e14964e7d09bc967d8c796e diff --git a/benchmarks/throughput_benchmark.cpp b/benchmarks/throughput_benchmark.cpp new file mode 100644 index 0000000..0620fec --- /dev/null +++ b/benchmarks/throughput_benchmark.cpp @@ -0,0 +1,289 @@ +#include +#include +#include +#include +#include +#include +#include + +// void print(const char *str) +// { +// write(STDOUT_FILENO, str, strlen(str)); +// } + +void isIntSumValid(int num, int sum) +{ + if (sum == ((num * (num - 1)) / 2)) + { + std::cout << "SUM is valid" << std::endl; + } + else + { + std::cout << "SUM is invalid" << std::endl; + } +} + +void isDoubleSumValid(double num, double sum) +{ + if (sum == ((num * (num - 1)) / 2)) + { + std::cout << "SUM is valid" << std::endl; + } + else + { + std::cout << "SUM is invalid" << std::endl; + } +} + +// Function to return the next random number +int getNum(std::vector &v) +{ + + // Size of the vector + int n = v.size(); + + // Generate a random number + srand(time(NULL)); + + // Make sure the number is within + // the index range + int index = rand() % n; + + // Get random number from the vector + int num = v[index]; + + // Remove the number from the vector + std::swap(v[index], v[n - 1]); + v.pop_back(); + + // Return the removed number + return num; +} + +// Function to generate n non-repeating random numbers +std::vector getIndexArray(int n) +{ + std::vector v(n); + + // Fill the vector with the values + // 1, 2, 3, ..., n + for (int i = 0; i < n; i++) + v[i] = i; + + return v; +} + +void bench_for_int(int num) +{ + int n = num * sizeof(int); + int sum = 0; + + int *p = (int *)malloc(n); + std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < (n / sizeof(int)); i++) + { + // printf("%d\n", i); + p[i] = i; + } + std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < (n / sizeof(int)); i++) + { + sum += p[i]; + } + std::chrono::high_resolution_clock::time_point t3 = std::chrono::high_resolution_clock::now(); + + std::chrono::duration total_time = std::chrono::duration_cast>(t3 - t1); + std::chrono::duration read_time = std::chrono::duration_cast>(t3 - t2); + std::chrono::duration write_time = std::chrono::duration_cast>(t2 - t1); + + std::cout << "--------------INT SEQ " << num << " ---------------" << std::endl; + + // write time + std::cout << "Write Time " << write_time.count() << " seconds." << std::endl; + // read time + std::cout << "Read Time " << read_time.count() << " seconds." << std::endl; + // total time + std::cout << "Total Time " << total_time.count() << " seconds." << std::endl; + // Avg read time / element + std::cout << "Avg Read Time " << read_time.count() / num << " seconds." << std::endl; + // Avg write time / element + std::cout << "Avg Write Time " << write_time.count() / num << " seconds." << std::endl; + // Avg total time / element + std::cout << "Avg Total Time " << total_time.count() / num << " seconds." << std::endl; + + // Sum + std::cout << "Sum " << sum << std::endl; + + isIntSumValid(num, sum); +} + +void bench_for_double(int num) +{ + int n = num * sizeof(double); + double sum = 0; + double j = 0; + + double *p = (double *)malloc(n); + std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < (n / sizeof(double)); i++) + { + // printf("%d\n", i); + p[i] = j; + j++; + } + std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); + + for (int i = 0; i < (n / sizeof(double)); i++) + { + sum += p[i]; + } + std::chrono::high_resolution_clock::time_point t3 = std::chrono::high_resolution_clock::now(); + + std::chrono::duration total_time = std::chrono::duration_cast>(t3 - t1); + std::chrono::duration read_time = std::chrono::duration_cast>(t3 - t2); + std::chrono::duration write_time = std::chrono::duration_cast>(t2 - t1); + + std::cout << "--------------DOUBLE SEQ " << num << " ---------------" << std::endl; + + // write time + std::cout << "Write Time " << write_time.count() << " seconds." << std::endl; + // read time + std::cout << "Read Time " << read_time.count() << " seconds." << std::endl; + // total time + std::cout << "Total Time " << total_time.count() << " seconds." << std::endl; + // Avg read time / element + std::cout << "Avg Read Time " << read_time.count() / num << " seconds." << std::endl; + // Avg write time / element + std::cout << "Avg Write Time " << write_time.count() / num << " seconds." << std::endl; + // Avg total time / element + std::cout << "Avg Total Time " << total_time.count() / num << " seconds." << std::endl; + + // Sum + std::cout << "Sum " << sum << std::endl; + isDoubleSumValid((double)num, sum); +} + +void bench_for_int_random(int num) +{ + int n = num * sizeof(int); + int sum = 0; + int i = 0; + + int *p = (int *)malloc(n); + std::vector idx1 = getIndexArray(num); + std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now(); + + while (idx1.size()) + { + i = getNum(idx1); + p[i] = i; + } + std::vector idx2 = getIndexArray(num); + + std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); + while (idx2.size()) + { + i = getNum(idx2); + sum += p[i]; + } + std::chrono::high_resolution_clock::time_point t3 = std::chrono::high_resolution_clock::now(); + + std::chrono::duration total_time = std::chrono::duration_cast>(t3 - t1); + std::chrono::duration read_time = std::chrono::duration_cast>(t3 - t2); + std::chrono::duration write_time = std::chrono::duration_cast>(t2 - t1); + + std::cout << "--------------INT RAND " << num << " ---------------" << std::endl; + + // write time + std::cout << "Write Time " << write_time.count() << " seconds." << std::endl; + // read time + std::cout << "Read Time " << read_time.count() << " seconds." << std::endl; + // total time + std::cout << "Total Time " << total_time.count() << " seconds." << std::endl; + // Avg read time / element + std::cout << "Avg Read Time " << read_time.count() / num << " seconds." << std::endl; + // Avg write time / element + std::cout << "Avg Write Time " << write_time.count() / num << " seconds." << std::endl; + // Avg total time / element + std::cout << "Avg Total Time " << total_time.count() / num << " seconds." << std::endl; + + // Sum + std::cout << "Sum " << sum << std::endl; + isIntSumValid(num, sum); +} + +void bench_for_double_random(int num) +{ + int n = num * sizeof(double); + int i = 0; + double sum = 0; + double j = 0; + + double *p = (double *)malloc(n); + std::vector idx1 = getIndexArray(num); + std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now(); + + while (idx1.size()) + { + i = getNum(idx1); + p[i] = i; + } + std::vector idx2 = getIndexArray(num); + + std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); + while (idx2.size()) + { + i = getNum(idx2); + sum += p[i]; + } + std::chrono::high_resolution_clock::time_point t3 = std::chrono::high_resolution_clock::now(); + + std::chrono::duration total_time = std::chrono::duration_cast>(t3 - t1); + std::chrono::duration read_time = std::chrono::duration_cast>(t3 - t2); + std::chrono::duration write_time = std::chrono::duration_cast>(t2 - t1); + + std::cout << "--------------DOUBLE RAND " << num << " ---------------" << std::endl; + + // write time + std::cout << "Write Time " << write_time.count() << " seconds." << std::endl; + // read time + std::cout << "Read Time " << read_time.count() << " seconds." << std::endl; + // total time + std::cout << "Total Time " << total_time.count() << " seconds." << std::endl; + // Avg read time / element + std::cout << "Avg Read Time " << read_time.count() / num << " seconds." << std::endl; + // Avg write time / element + std::cout << "Avg Write Time " << write_time.count() / num << " seconds." << std::endl; + // Avg total time / element + std::cout << "Avg Total Time " << total_time.count() / num << " seconds." << std::endl; + + // Sum + std::cout << "Sum " << sum << std::endl; + isDoubleSumValid((double)num, sum); +} + +int main(int argc, char *argv[]) +{ + // bench_for_int(1024); + // bench_for_int(10240); + // bench_for_int(102400); + // bench_for_int(1024000); + + // bench_for_double(1024); + // bench_for_double(10240); + // bench_for_double(102400); + // bench_for_double(1024000); + + bench_for_int_random(1024); + bench_for_int_random(10240); + bench_for_int_random(102400); + bench_for_int_random(1024000); + + bench_for_double_random(1024); + bench_for_double_random(10240); + bench_for_double_random(102400); + bench_for_double_random(1024000); +} \ No newline at end of file diff --git a/client/Makefile b/client/Makefile new file mode 100644 index 0000000..521d74d --- /dev/null +++ b/client/Makefile @@ -0,0 +1,36 @@ +CXX=g++ +INC_DIR = ../include +CXXFLAGS=-c -Wall -I $(INC_DIR) # Ugly! +SRC=../uffdman/*.cpp + +all: client #preloadlib + +# preloadlib: uffdman.o +# g++ bin/uffdman.o bin/rmp_client.o -shared -fPIC -o ./bin/preloadlib.so preloadlib.cpp -ldl -lpthread + +client: uffdman.o + $(CXX) -o bin/client -pthread bin/uffdman.o bin/rmp_client.o bin/client.o + +#rmp_client.cpp ../uffdman/uffdman.cpp client.cpp + + +# rmp_test: rmp_test.o +# $(CXX) -o bin/rmp_test -pthread bin/rmp_test.o bin/rmp_server.o + +# rmp_test.o: rmp_server.o +# $(CXX) -c rmp_test.cpp -o bin/rmp_test.o + +uffdman.o: rmp_client.o + $(CXX) $(CXXFLAGS) ../uffdman/uffdman.cpp -o bin/uffdman.o + +rmp_client.o: client.o + $(CXX) $(CXXFLAGS) rmp_client.cpp -o bin/rmp_client.o + +client.o: mkdir + $(CXX) $(CXXFLAGS) client.cpp -o bin/client.o + +mkdir: + mkdir -p bin + +clean: + rm -rf bin diff --git a/client/client.cpp b/client/client.cpp index 9943fd6..c666eac 100644 --- a/client/client.cpp +++ b/client/client.cpp @@ -1,44 +1,75 @@ -// Client side C/C++ program to demonstrate Socket programming - -#include -#include -#include -#include -#include -#define PORT 8080 - -int main(int argc, char const *argv[]) -{ - int sock = 0, valread; - struct sockaddr_in serv_addr; - const char *alloc_req = "100"; - char buffer[1024] = {0}; - if ((sock = socket(AF_INET, SOCK_STREAM, 0)) < 0) - { - printf("\n Socket creation error \n"); - return -1; - } - - serv_addr.sin_family = AF_INET; - serv_addr.sin_port = htons(PORT); - - // Convert IPv4 and IPv6 addresses from text to binary form - if(inet_pton(AF_INET, "127.0.0.1", &serv_addr.sin_addr)<=0) - { - printf("\nInvalid address/ Address not supported \n"); - return -1; - } - - if (connect(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) - { - printf("\nConnection Failed \n"); - return -1; - } - - send(sock, alloc_req, strlen(alloc_req), 0); - printf("Request sent for page size: %s\n", alloc_req); - - valread = read(sock, buffer, 1024); - printf("Fd recieved at client %s\n", buffer); - return 0; -} +#include +#include +#include +#include "rmp_client.hpp" + +#define PAGE_SIZE sysconf(_SC_PAGE_SIZE) + +int main(int argc, char const *argv[]) +{ + double sum = 0; + double sum2 = 0; + double sum3 = 0; + if (argc != 3) + { + std::cout << "Invalid args!" << std::endl; + std::cout << "Usage: ./server " << std::endl; + return 1; + } + + std::cout << argv[1] << std::endl; + std::cout << atoi(argv[2]) << std::endl; + + rmp::config cnf{ + argv[1], + atoi(argv[2])}; + + rmp::Client client(cnf); + + printf(" RMP_INIT returned %d\n ", client.rmp_init()); + + double *data = (double *)client.rmp_alloc(11); + double *data2 = (double *)client.rmp_alloc(11); + double *data3 = (double *)client.rmp_alloc(11); + + printf("data 1 : %lx\n", data); + printf("data 2 : %lx\n", data2); + printf("data 3 : %lx\n", data3); + + for (int i = 0; i < ((10 * PAGE_SIZE) / sizeof(double)); i++) + { + data[i] = i; + } + + for (int i = 0; i < ((10 * PAGE_SIZE) / sizeof(double)); i++) + { + printf("Value at %lx is %f\n", (data + i), data[i]); + sum += data[i]; + } + + for (int i = 0; i < ((10 * PAGE_SIZE) / sizeof(double)); i++) + { + data2[i] = i; + } + + for (int i = 0; i < ((10 * PAGE_SIZE) / sizeof(double)); i++) + { + printf("Value at %lx is %f\n", (data2 + i), data2[i]); + sum2 += data2[i]; + } + + for (int i = 0; i < ((10 * PAGE_SIZE) / sizeof(double)); i++) + { + data3[i] = i; + } + + for (int i = 0; i < ((10 * PAGE_SIZE) / sizeof(double)); i++) + { + printf("Value at %lx is %f\n", (data3 + i), data3[i]); + sum3 += data3[i]; + } + + printf("Sum 1 : %f\n", sum); + printf("Sum 2 : %f\n", sum2); + printf("Sum 3 : %f\n", sum3); +} \ No newline at end of file diff --git a/client/rmp_client.cpp b/client/rmp_client.cpp new file mode 100644 index 0000000..043241b --- /dev/null +++ b/client/rmp_client.cpp @@ -0,0 +1,275 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "uffdman.hpp" +#include "rmp_types.hpp" +#include "rmp_client.hpp" + +#define PAGE_SIZE sysconf(_SC_PAGE_SIZE) + +#ifndef DEBUG_PRINTS +#define DEBUG_PRINTS 0 +#endif + +#if DEBUG_PRINTS +#define err(msg) perror(msg) +#define printf(exp...) printf(exp); +#else +#define err(msg) +#define printf(exp...) +#endif + +char *print_stmt = ""; +int socketfd; +bool connected = false; +std::unordered_map *addr_hndl_map; +std::unordered_map *addr_npages_map; + +void print(int n) +{ + // If number is smaller than 0, put a - sign + // and change number to positive + if (n < 0) + { + write(STDOUT_FILENO, "-", 1); + n = -n; + } + + // Remove the last digit and recur + if (n / 10) + print(n / 10); + + char s = ((n % 10) + '0'); + // Print the last digit + write(STDOUT_FILENO, &s, 1); +} + +void rmp_write_page(rmp::handle hd, ul offset, void *page) +{ + + // print_stmt = "Entered rmp_write_page\n"; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); + + if (!connected) + { + return; + } + + int *pg = (int *)page; + for (int i = 0; i < 4096 / sizeof(int); i++) + { + printf("Write Value : %d\n", pg[i]); + } + + rmp::packet send_pkt; + send_pkt.action = 2; + memcpy(send_pkt.data, page, PAGE_SIZE); + send_pkt.size = PAGE_SIZE; + send_pkt.offset = (u32)offset; + send_pkt.error = 0; + send_pkt.hndl = hd; + + rmp::packet return_pkt; + + // print_stmt = "SEND BYTES (rmp_write_page): "; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); + // print((int)send(socketfd, &send_pkt, sizeof(rmp::packet), 0)); + send(socketfd, &send_pkt, sizeof(rmp::packet), 0); + usleep(1000); + + // print_stmt = "\nSent req (rmp_write_page)\n"; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); + + recv(socketfd, &return_pkt, sizeof(rmp::packet), 0); + usleep(1000); + + // print_stmt = "Recvd req (rmp_write_page)\n"; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); + + printf("Error (rmp_write_page) : %u\n", return_pkt.error); + + // print_stmt = "Exiting rmp_write_page\n"; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); +} + +void rmp_read_page(rmp::handle hd, ul offset, void *page) +{ + // print_stmt = "Entered rmp_read_page\n"; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); + + if (!connected) + { + return; + } + + rmp::packet send_pkt; + send_pkt.action = 1; + send_pkt.offset = offset; + send_pkt.size = PAGE_SIZE; + send_pkt.error = 0; + send_pkt.hndl = hd; + + rmp::packet return_pkt; + + // print_stmt = "SEND BYTES (rmp_read_page) : "; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); + // print((int)send(socketfd, &send_pkt, sizeof(rmp::packet), 0)); + send(socketfd, &send_pkt, sizeof(rmp::packet), 0); + usleep(1000); + + // print_stmt = "\nSent req (rmp_read_page)\n"; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); + + recv(socketfd, &return_pkt, sizeof(rmp::packet), 0); + usleep(1000); + + // print_stmt = "Recvd req (rmp_read_page)\n"; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); + + memcpy(page, return_pkt.data, PAGE_SIZE); + + int *pg = (int *)page; + for (int i = 0; i < 4096 / sizeof(int); i++) + { + printf("Read Value : %d\n", pg[i]); + } + + // print_stmt = "Exiting rmp_read_page\n"; + write(STDOUT_FILENO, print_stmt, strlen(print_stmt)); +} + +void rmp_pagefault_resovler(void *start_addr, void *faulting_addr, int is_write, void *page) +{ + ul offset = (((ul)faulting_addr & ~(PAGE_SIZE - 1)) - (ul)start_addr); + ul page_offset = floor(offset / PAGE_SIZE); + rmp::handle hndl = addr_hndl_map->at((ul)start_addr); + printf("rmp_pagefault_resovler: "); + printf("Offset = %ld; ", offset); + printf("Page Offset = %ld; ", page_offset); + printf("Start Address = %" PRIx64 "; ", start_addr); + printf("Faulting Address = %" PRIx64 "\n", faulting_addr); + if (is_write) + { + rmp_write_page(hndl, page_offset, page); + } + else + { + rmp_read_page(hndl, page_offset, page); + } +} + +// Setup the args to establish connection +rmp::Client::Client(rmp::config cnf) +{ + this->server_config = cnf; +} + +int rmp::Client::rmp_init() +{ + int valread; + struct sockaddr_in serv_addr; + if ((socketfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) + { + printf("\n Socket creation error \n"); + return -1; + } + + serv_addr.sin_family = AF_INET; + serv_addr.sin_addr.s_addr = inet_addr(this->server_config.addr.c_str()); + serv_addr.sin_port = htons(this->server_config.port); + + // // Convert IPv4 and IPv6 addresses from text to binary form + // if (inet_pton(AF_INET, "127.0.0.1", &serv_addr.sin_addr) <= 0) + // { + // printf("\nInvalid address/ Address not supported \n"); + // return -1; + // } + + if (connect(socketfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) + { + printf("\nConnection Failed \n"); + return -1; + } + else + { + connected = true; + } + + if (socketfd == -1) + { + printf("Error while establishing connection to server\n"); + return -1; + } + + uffdman_init(); + uffdman_register_page_resolver(&rmp_pagefault_resovler); + + // Initialize the maps + addr_hndl_map = new std::unordered_map(); + addr_npages_map = new std::unordered_map(); + + return 0; +} + +void *rmp::Client::rmp_alloc(u32 npages) +{ + long size = npages * PAGE_SIZE; + void *new_addr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + if (new_addr == MAP_FAILED) + { + printf("Error while allocating memory on client"); + return (char *)MAP_FAILED; + } + + printf("socketfd : %d, connected : %d\n", socketfd, connected); + if (!connected) + { + return NULL; + } + + rmp::packet send_pkt; + send_pkt.action = 0; + send_pkt.npages = npages; + send_pkt.size = PAGE_SIZE; + + rmp::packet return_pkt; + + send(socketfd, &send_pkt, sizeof(rmp::packet), 0); + usleep(1000); + + recv(socketfd, &return_pkt, sizeof(rmp::packet), 0); + usleep(1000); + + printf("Handle : %d\n", return_pkt.hndl); + + uffdman_register_region(new_addr, npages); + + ul ul_addr = (ul)new_addr; + addr_hndl_map->insert({ul_addr, return_pkt.hndl}); + addr_npages_map->insert({ul_addr, npages}); + + return new_addr; +} + +int rmp::Client::rmp_free(rmp::handle hndl) +{ +} + +void rmp::Client::rmp_read(rmp::handle hd, ul offset, void *page) +{ + rmp_read_page(hd, offset, page); +} + +void rmp::Client::rmp_write(rmp::handle hd, ul offset, void *page) +{ + rmp_write_page(hd, offset, page); +} \ No newline at end of file diff --git a/example.c b/example.c deleted file mode 100644 index fee150f..0000000 --- a/example.c +++ /dev/null @@ -1,52 +0,0 @@ - - -// client -// - -static void pf_handler (uint64_t faulting_address) -{ - // is it a special remote address? - handle * handle = lookup_in_handle_hashtable(faulting_address); - if (handle) { - // this is a remote region - // - if (fault_type == FAULT_READ) { - read_page(handle); - } else { // it's a write - write_page(handle); - } - - // OTHERWISE ERROR - - } - -} - -static char * alloc_remote_mem(size_t size) { - // use your remote API here - handle * handle = get_new_remote_page(); - char * dummy_addr = mmap(....,PROT_NONE,...); - insert_handle_into_hashtable(dummy_addr, handle); - return dummy_addr; -} - -static void read_page (handle * handle, size_t offset, size_t len) -{ - // use your remote API here -} - -static void write_page (handle * handle, size_t offset, size_t len, void * src) -{ - // use your remote API here -} - - -int main () { - register_userfaultfd_handler(pf_handler); - - char * remote = alloc_remote_mem(4096); // this needs to give something back which IS NOT MAPPED - *remote = 0xdeadbeef; // this will cause a page fault, which will vector to pf_handler - - free_remote_mem(remote); -} - diff --git a/include/rmp_client.hpp b/include/rmp_client.hpp new file mode 100644 index 0000000..0801f0b --- /dev/null +++ b/include/rmp_client.hpp @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include +#include +#include "rmp_types.hpp" + +namespace rmp +{ + + // Request contains fields required to + // complete a client's request + typedef struct + { + handle hndl; + u32 npages; + u32 size; + u32 offset; + u32 action; // 0 -> map, 1 -> get, 2 -> put, 3 -> free + char data[4096]; // Data to be read or written + u32 error; + } packet; + + typedef struct + { + /* + TODO: Define a commong connection configuration for both guest and host + + Example: IPv4, Protocol, Port, etc. + */ + std::string addr; + ul port; + } config; + + class Client + { + private: + u32 handle_count = 0; + config server_config; + std::vector handles; + + public: + Client(config); // Setup the args to establish connection + int rmp_init(); // Connect to Server + void *rmp_alloc(u32); // Allocate n pages on server and returns a handle + int rmp_free(rmp::handle); //free pages allocated for the handle + void rmp_read(rmp::handle hd, ul offset, void *page); // Read a page offset from a specific handle + void rmp_write(rmp::handle hd, ul offset, void *page); // Write a page at offset for a specific handle + + // inline int get_offset(ul addr, ul fault_addr, u32 page_size) + // { + // for (int i = 0; i < handles.size(); i++) + // { + // rmp::handle hndl = this->handles[i]; + // u32 npages = this->addr_npages_map[i]; + // ul start_addr = this->addr_hndl_map[hndl]; + // ul end_addr = start_addr + (npages * page_size); + // if (fault_addr >= start_addr && fault_addr <= end_addr) + // { + // u32 offset = std::floor((fault_addr - start_addr) / page_size); + // return offset; + // } + // } + // return -1; + // } + }; +} // namespace rmp \ No newline at end of file diff --git a/include/rmp_server.hpp b/include/rmp_server.hpp new file mode 100644 index 0000000..312279a --- /dev/null +++ b/include/rmp_server.hpp @@ -0,0 +1,106 @@ +#pragma once + +#include +#include +#include "rmp_types.hpp" + +namespace rmp +{ + // Request contains fields required to + // complete a client's request + typedef struct + { + handle hndl; + u32 npages; + u32 size; + u32 offset; + u32 action; // 0 -> map, 1 -> get, 2 -> put, 3 -> free + char data[4096]; // Data to be read or written + u32 error; + } packet; + + typedef struct + { + /* + TODO: Define a commong connection configuration for both guest and host + + Example: IPv4, Protocol, Port, etc. + */ + } config; + + class Server + { + private: + u32 handle_count = 0; + config server_config; + std::unordered_map addr_map; + std::unordered_map hndl_n_pages; + std::mutex handle_lock, addr_map_lock, hndl_n_pages_lock; + + public: + Server(rmp::config) + { + } + + void handle(rmp::packet *); + rmp::handle alloc(u32, u32); + int read(rmp::handle, rmp::packet *); + int write(rmp::handle handle, rmp::packet *packet); + int free(rmp::handle, u32); + + inline rmp::handle get_new_handle() + { + std::lock_guard guard(this->handle_lock); + u32 new_handle = handle_count++; + return new_handle; + } + + inline void set_addr_map(rmp::handle hndl, void *addr) + { + std::lock_guard guard(this->addr_map_lock); + this->addr_map[hndl] = addr; + } + + inline void *get_addr_map(rmp::handle hndl) + { + std::lock_guard guard(this->addr_map_lock); + return this->addr_map[hndl]; + } + + inline void unset_addr_map(rmp::handle hndl) + { + std::lock_guard guard(this->addr_map_lock); + this->addr_map.erase(hndl); + } + + inline void set_hndl_n_pages(rmp::handle hndl, u32 num_pages) + { + std::lock_guard guard(this->hndl_n_pages_lock); + this->hndl_n_pages[hndl] = num_pages; + } + + inline u32 get_hndl_n_pages(rmp::handle hndl) + { + std::lock_guard guard(this->hndl_n_pages_lock); + return this->hndl_n_pages[hndl]; + } + + inline void unset_hndl_n_pages(rmp::handle hndl) + { + std::lock_guard guard(this->hndl_n_pages_lock); + this->hndl_n_pages.erase(hndl); + } + + inline bool contains_key(rmp::handle handle) + { + std::lock_guard guard(this->addr_map_lock); + return this->addr_map.find(handle) != this->addr_map.end(); + } + }; + + typedef struct + { + int client_sock; + rmp::Server *server; + } thread_req; +} // namespace rmp \ No newline at end of file diff --git a/include/rmp_types.hpp b/include/rmp_types.hpp new file mode 100644 index 0000000..ec6b045 --- /dev/null +++ b/include/rmp_types.hpp @@ -0,0 +1,34 @@ +#pragma once + +typedef unsigned int u32; +typedef unsigned long ul; + +static u32 ALLOC_FAILED = 1; +static u32 READ_FAILED = 2; +static u32 WRITE_FAILED = 3; +static u32 FREE_FAILED = 4; +static u32 UNKNOWN_ACTION = 5; +static u32 INVALID_HANDLE = 6; + +namespace rmp +{ + typedef long handle; + + // enum req_type + // { + // ALLOC_PAGES, + // READ_PAGE, + // WRITE_PAGE, + // FREE_PAGES + // } rmp_req; + + // enum error_type + // { + // ALLOC_FAILED, + // READ_FAILED, + // WRITE_FAILED, + // FREE_FAILED, + // UNKNOWN_ACTION, + // INVALID_HANDLE + // } rmp_err; +} // namespace rmp diff --git a/include/uffdman.hpp b/include/uffdman.hpp new file mode 100644 index 0000000..b16cea6 --- /dev/null +++ b/include/uffdman.hpp @@ -0,0 +1,13 @@ +#pragma once + +void uffdman_init(); + +void uffdman_destroy(); + +void uffdman_register_page_resolver(void (*handler)(void *start_addr, void *faulting_addr, int is_write, void *page)); + +void uffdman_unregister_page_resolver(); + +int uffdman_register_region(void *addr, unsigned long n_pages); + +void uffdman_unregister_region(void *addr); diff --git a/preloadlib/Makefile b/preloadlib/Makefile new file mode 100644 index 0000000..6cd8289 --- /dev/null +++ b/preloadlib/Makefile @@ -0,0 +1,20 @@ +SRC=../uffdman/uffdman.cpp ../client/rmp_client.cpp +# SRC=../rmp.so + +all: preloadlib preload-tests + +preloadlib: mkdir + g++ -Wall -I ../include $(SRC) -shared -fPIC -o ./bin/preloadlib.so preloadlib.cpp -ldl -lpthread + +preload-tests: + $(MAKE) -C tests/ all + +mkdir: + mkdir -p bin + +clean: clean-tests + rm -rf bin + +clean-tests: + $(MAKE) -C tests/ clean + diff --git a/preloadlib/preloadlib.cpp b/preloadlib/preloadlib.cpp new file mode 100644 index 0000000..40c67f0 --- /dev/null +++ b/preloadlib/preloadlib.cpp @@ -0,0 +1,208 @@ +// #ifndef _GNU_SOURCE +// #define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rmp_client.hpp" + +// #if DEBUG_PRINTS +// #define err(msg) perror(msg) +// #define cout(exp) cout << exp; +// #else +// #define err(msg) +// #define cout(exp) +// #endif + +#define PAGE_SIZE sysconf(_SC_PAGE_SIZE) + +static void *(*stdlib_malloc)(size_t) = NULL; +static void *(*stdlib_realloc)(void *, size_t) = NULL; +static void *(*stdlib_calloc)(size_t, size_t) = NULL; +static void (*stdlib_free)(void *) = NULL; + +static int preloadlib_init_pending = 0; +static int do_local_allocation = 0; + +int rmp_fd = -1; +pthread_t mainthread_id; +rmp::Client *client; +extern std::mutex fault_handler_cr; + +void print(const char *str) +{ + // #if DEBUG_PRINTS + write(STDOUT_FILENO, str, strlen(str)); + // #endif +} + +// void myprintf(const char *__restrict__ __format, ...) +// { +// va_list argptr; +// va_start(argptr, __format); +// vfprintf(stderr, __format, argptr); +// va_end(argptr); +// } + +// Initialize the rmp connection to server and get a fd +static void rmp_conn_init(void) +{ + print("preloadlib.so: (init) called\n"); + + // rmp::config conf{ + // "127.0.0.1", + // 6767}; + + rmp::config conf{ + "127.0.0.1", + 6768}; + + std::lock_guard guard(fault_handler_cr); + do_local_allocation = 1; + mainthread_id = pthread_self(); + print("preloadlib.so: (init) rmp_init Started\n"); + client = new rmp::Client(conf); + rmp_fd = client->rmp_init(); + // printf("RMP_FD IS %d\n", rmp_fd); + print("preloadlib.so: (init) rmp_init finished\n"); + do_local_allocation = 0; + print("preloadlib.so: (init) exiting\n"); +} + +/* Load original allocation routines at first use */ +static void preloadlib_init(void) +{ + print("preloadlib.so: (preloadlib_init) called\n"); + preloadlib_init_pending = 1; + stdlib_malloc = (void *(*)(size_t))dlsym(RTLD_NEXT, "malloc"); + stdlib_realloc = (void *(*)(void *, size_t))dlsym(RTLD_NEXT, "realloc"); + stdlib_calloc = (void *(*)(size_t, size_t))dlsym(RTLD_NEXT, "calloc"); + stdlib_free = (void (*)(void *))dlsym(RTLD_NEXT, "free"); + if (!stdlib_malloc || !stdlib_realloc || !stdlib_calloc || !stdlib_free) + { + print("preloadlib.so: Unable to hook allocation!\n"); + print(dlerror()); + exit(1); + } + else + { + print("preloadlib.so: Successfully hooked\n"); + } + + // Initialize the rmp connection + // rmp_conn_init(); + + preloadlib_init_pending = 0; + print("preloadlib.so: (preloadlib_init) exiting\n"); +} + +static int check_conn(void) +{ + print("preloadlib.so: (check_conn) called\n"); + // print("preloadlib.so: (check_conn) called\n"); + if (!stdlib_malloc) + { + preloadlib_init(); + } + + if (rmp_fd == -1) + { + rmp_conn_init(); + } + + if (rmp_fd != -1) + { + print("rmp_fd is initialized!\n"); + } + else + { + print("rmp_fd is uninitialized!\n"); + } + print("preloadlib.so: (check_conn) exiting\n"); + return rmp_fd != -1; + // return false; +} + +void free(void *ptr) +{ + print("preloadlib.so: (free) called\n"); + // if (do_local_allocation == 1) + // { + // print("Local free\n"); + // // stdlib_free(ptr); + // } + // else + // { + // if (check_conn()) + // { + // print("Remote free\n"); + // do_local_allocation = 1; + // rmp_free((char *)ptr); + // do_local_allocation = 0; + // } + // } + print("preloadlib.so: (free) exiting\n"); +} + +void *local_malloc(size_t size) +{ + void *result; + print("preloadlib.so: (malloc) local allocation started\n"); + if (stdlib_malloc == NULL) + { + preloadlib_init(); + print("preloadlib.so: (malloc) stdlib_malloc is NULL\n"); + return NULL; + } + result = stdlib_malloc(size); + print("preloadlib.so: (malloc) local allocation done\n"); + return result; +} + +void *malloc(size_t size) +{ + print("preloadlib.so: (malloc) called\n"); + + void *result; + if (do_local_allocation) + { + return local_malloc(size); + } + + if (check_conn()) + { + + if (pthread_self() != mainthread_id) + { + return local_malloc(size); + } + + std::lock_guard guard(fault_handler_cr); + //call rmp_alloc + print("preloadlib.so: (malloc) remote allocation\n"); + do_local_allocation = 1; + long npages = ceil((float)size / (float)PAGE_SIZE); + // printf("Size : %ld, PAGE SIZE : %ld, Pages : %ld\n", size, PAGE_SIZE, npages); + result = (void *)client->rmp_alloc(npages); + fault_handler_cr.unlock(); + print("preloadlib.so: (malloc) remote allocation finished\n"); + do_local_allocation = 0; + return result; + } + // else + // { + // result = stdlib_malloc(size); + // } + + //fprintf(stderr, "preloadlib.so: malloc(0x%zx) = %p\n", size, result); + print("preloadlib.so: (malloc) failed\n"); + return NULL; +} \ No newline at end of file diff --git a/preloadlib/tests/Makefile b/preloadlib/tests/Makefile new file mode 100644 index 0000000..a5020b5 --- /dev/null +++ b/preloadlib/tests/Makefile @@ -0,0 +1,18 @@ + +all: malloc_test calloc_test realloc_test + +malloc_test: mkdir + gcc -g malloc_test.c -o ./bin/malloc_test + +calloc_test: mkdir + gcc -g calloc_test.c -o ./bin/calloc_test + +realloc_test: mkdir + gcc -g realloc_test.c -o ./bin/realloc_test + +mkdir: + mkdir -p bin + +clean: + rm -rf bin + \ No newline at end of file diff --git a/preloadlib/tests/calloc_test.c b/preloadlib/tests/calloc_test.c new file mode 100644 index 0000000..982af9b --- /dev/null +++ b/preloadlib/tests/calloc_test.c @@ -0,0 +1,30 @@ +#include +#include +#include +#include + +void print(const char *str) +{ + write(STDOUT_FILENO, str, strlen(str)); +} + +int main(int argc, char *argv[]) +{ + int i; + const int limit = 5; + void *calloc_ptrs[limit]; + + printf("%s\n", "calloc_test: (main) called"); + + for (i = 0; i < limit; ++i) + { + calloc_ptrs[i] = calloc(10000 * i, sizeof(char)); + } + + for (i = 0; i < limit; ++i) + { + free(calloc_ptrs[i]); + } + printf("%s\n", "calloc_test: (main) success, exiting"); + return 0; +} \ No newline at end of file diff --git a/preloadlib/tests/malloc_test.c b/preloadlib/tests/malloc_test.c new file mode 100644 index 0000000..6af48ef --- /dev/null +++ b/preloadlib/tests/malloc_test.c @@ -0,0 +1,90 @@ +#include +#include +#include +#include + +void print(const char *str) +{ + write(STDOUT_FILENO, str, strlen(str)); +} + +static int foo() +{ + int n = 4096 * 100; + int sum = 0; + + int *p = (int *)malloc(n); + + // memset(p, 1, n); + for (int i = 0; i < (n / sizeof(int)); i++) + { + // write(STDOUT_FILENO, i + '0', strlen(i + '0')); + // write(STDOUT_FILENO, '\n', 1); + printf("%d\n", i); + p[i] = i; + } + + for (int i = 0; i < (n / sizeof(int)); i++) + { + sum += p[i]; + } + + // // memset(p, 1, n); + // for (int i = 0; i < (n / sizeof(int)) / 2; i++) + // { + // // write(STDOUT_FILENO, i + '0', strlen(i + '0')); + // // write(STDOUT_FILENO, '\n', 1); + // printf("%d\n", i); + // p[i] = i; + // } + + // for (int i = 0; i < (n / sizeof(int)) / 2; i++) + // { + // sum += p[i]; + // } + + // // memset(p, 1, n); + // for (int i = (n / sizeof(int)) / 2; i < (n / sizeof(int)); i++) + // { + // // write(STDOUT_FILENO, i + '0', strlen(i + '0')); + // // write(STDOUT_FILENO, '\n', 1); + // printf("%d\n", i); + // p[i] = i; + // } + + // for (int i = (n / sizeof(int)) / 2; i < (n / sizeof(int)); i++) + // { + // sum += p[i]; + // } + return sum; +} + +int main(int argc, char *argv[]) +{ + // int i; + // const int limit = 100; + // void *malloc_ptrs[limit]; + // getchar(); + // printf("Hello World!\n"); + + printf("%d\n", foo()); +#if 0 + printf("%s\n", "malloc_test: (main) called"); + for (i = 0; i < limit; ++i) + { + malloc_ptrs[i] = malloc(10000 * i); + } + + for (i = 0; i < limit; ++i) + { + *((char *)malloc_ptrs[i]) = 'H'; + } + + for (i = 0; i < limit; ++i) + { + free(malloc_ptrs[i]); + } + printf("%s\n", "malloc_test: (main) success, exiting"); +#endif + return 0; +} \ No newline at end of file diff --git a/preloadlib/tests/realloc_test.c b/preloadlib/tests/realloc_test.c new file mode 100644 index 0000000..f524add --- /dev/null +++ b/preloadlib/tests/realloc_test.c @@ -0,0 +1,34 @@ +#include +#include +#include +#include + +void print(const char *str) +{ + write(STDOUT_FILENO, str, strlen(str)); +} + +int main(int argc, char *argv[]) +{ + int i; + const int limit = 5; + void *malloc_ptrs[limit]; + + printf("%s\n", "realloc_test: (main) called"); + for (i = 0; i < limit; ++i) + { + malloc_ptrs[i] = malloc(10000 * i); + } + + for (i = 0; i < limit; ++i) + { + malloc_ptrs[i] = realloc(malloc_ptrs[i], 20000 * i); + } + + for (i = 0; i < limit; ++i) + { + free(malloc_ptrs[i]); + } + printf("%s\n", "realloc_test: (main) success, exiting"); + return 0; +} \ No newline at end of file diff --git a/rmp.cpp b/rmp.cpp deleted file mode 100644 index 296472b..0000000 --- a/rmp.cpp +++ /dev/null @@ -1,169 +0,0 @@ -#include -#include -#include -#include -#include - -/* - Potential TODO: - - convert guest and host to classes - - useful for multiple instances of hosts/guests. - - Guests can have proxy host objects -*/ -namespace rmp -{ - namespace guest - { - static struct config host_conf; - - static unordered_map addr_hndl_map; - static unordered_map addr_npages_map; - - void rmp_init(struct config conf) { - host_conf = conf; - // TODO connect to host - register_uffd_page_resolver(&rmp_pagefault_resolver); - } - - char* rmp_alloc(ul n_pages) - { - // new invalid address at guest side to cause pagefault upon it's access - char* new_invalid_addr = mmap(NULL, 0, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - - ul ul_addr = (ul) new_invalid_addr; - - // TODO request for new allocation of n_pages - - // TODO receives a rmp_handle from host - rmp_handle server_hndl; - - register_uffd_region(new_invalid_addr, n_pages); - - addr_hndl_map[ul_addr] = server_hndl; - addr_npages_map[ul_addr] = n_pages; - - return new_invalid_addr; - } - - void rmp_free(char* addr) - { - rmp_handle hndl = addr_hndl_map[(ul) addr]; - // TODO send free request for hndl to server - // TODO free the invalid address that is stored in the table - - // remove stored references of address and size - addr_hndl_map.erase((ul) addr); - addr_npages_map.erase((ul) addr); - } - - // private guest - void rmp_pagefault_resovler(char* start_addr, char* faulting_addr, long page_size, int is_write, char** page) - { - ul offset = (ul) ((faulting_addr - start_addr) / page_size); - if(offset <= addr_npages_map[start_addr]) { - rmp_handle hndl = addr_hndl_map[start_addr]; - if(is_write) - { - rmp_write(hndl, offset, *page); - } - else - { - *page = rmp_read(hndl, offset); - } - } else { - // TODO out of memory region - } - } - - static void rmp_write_page(rmp_handle hd, ul offset, char* page) - { - // TODO send page to server with write request - } - - static char* rmp_read_page(rmp_handle hd, ul offset) - { - char* page; - // TODO receive page from server by sending a read request - return page; - } - } - - namespace host - { - static struct confing guest_conf; - static rmp_hanle alloc_count = 0; - static long page_size = -1; - - static unordered_map hndl_addr_map; - static unordered_map addr_npages_map; - - // Host - void rmp_host_init(struct config conf) - { - // restrict incoming/outgoing requests/responses to only given guest - guest_conf = conf; - - page_size = sysconf(_SC_PAGE_SIZE); - - // TODO start a host w/ given config - - /* - TODO hook incoming requests w/ functions below - */ - - // TODO print host config using rmp_host_config to console - - } - - struct config rmp_host_config() { - struct config host_conf; - // TODO Fill host configuration - return host_conf; - } - - static rmp_handle rmp_host_alloc(long n_pages) - { - long mem_region_size = n_pages * page_size; - - char* remote_mem_region = mmap(NULL, mem_region_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - - if(remote_mem_region == MAP_FAILED) - { - // TODO Handle error - } - - ++alloc_count; - ul ul_addr = (ul) remote_mem_region; - - hndl_addr_map[alloc_count] = ul_addr; - addr_npages[ul_addr] = np_pages; - - return alloc_count; - } - - static void rmp_host_free(rmp_handle hndl) - { - ul addr = hdnl_addr_map[hndl]; - ul n_pages = addr_npages_map[addr]; - long size = n_pages * page_size; - if (munmap(addr, size) == -1) - { - // TODO Handle error - } - hdnl_addr_map.erase(hndl); - addr_npages_map.erase(addr); - } - - static void rmp_host_write(rmp_handle hndl, long offset, char* page) - { - char* addr = (char*) hndl_addr_map[hndl]; - *(addr + (char*) offset) = *(page); - } - - static char* rmp_host_read(rmp_handle hndl, long offset) - { - char* addr = (char*) hndl_addr_map[hndl]; - return (addr + (char*) offset); - } - } -} diff --git a/rmp.hpp b/rmp.hpp deleted file mode 100644 index cc7bebc..0000000 --- a/rmp.hpp +++ /dev/null @@ -1,35 +0,0 @@ -#ifdef RMP_H -#define RMP_H - -namespace rmp -{ - - typedef int rmp_handle; - typedef unsigned long ul; - - struct config - { - /* - TODO: Define a commong connection configuration for both guest and host - - Example: IPv4, Protocol, Port, etc. - */ - }; - - namespace guest - { - // Guest - void rmp_init(struct config); - char* rmp_alloc(long); // receives a rmp_handle from host - void rmp_free(rmp_handle); - } - - namespace host - { - // Host - void rmp_host_setup(struct config); - struct config rmp_host_config(); - } -} - -#endif diff --git a/server/.DS_Store b/server/.DS_Store new file mode 100644 index 0000000..8266462 Binary files /dev/null and b/server/.DS_Store differ diff --git a/server/Makefile b/server/Makefile new file mode 100644 index 0000000..2214f00 --- /dev/null +++ b/server/Makefile @@ -0,0 +1,29 @@ +CXX=g++ +INC_DIR = ../include +CXXFLAGS=-c -Wall -I$(INC_DIR) # Ugly! +DEPS = $(INC_DIR)/rmp.hpp + +all: server + +server: server.o + $(CXX) -o bin/server -pthread bin/server.o bin/rmp_server.o + +server.o: rmp_test + $(CXX) $(CXXFLAGS) -c server.cpp -o bin/server.o + +rmp_test: rmp_test.o + $(CXX) -o bin/rmp_test -pthread bin/rmp_test.o bin/rmp_server.o + +rmp_test.o: rmp_server.o + $(CXX) $(CXXFLAGS) rmp_test.cpp -o bin/rmp_test.o + +rmp_server.o: mkdir + $(CXX) $(CXXFLAGS) rmp_server.cpp -o bin/rmp_server.o + + + +mkdir: + mkdir -p bin + +clean: + rm -rf bin diff --git a/server/out b/server/out new file mode 100644 index 0000000..eff4f56 --- /dev/null +++ b/server/out @@ -0,0 +1,2 @@ +10.237.15.93 +6768 diff --git a/server/rmp_server.cpp b/server/rmp_server.cpp new file mode 100644 index 0000000..e02c66e --- /dev/null +++ b/server/rmp_server.cpp @@ -0,0 +1,170 @@ +#include +#include +#include "rmp_types.hpp" +#include "rmp_server.hpp" + +void handle_alloc(rmp::Server *rmp_server, rmp::packet *packet) +{ + + rmp::handle new_hndl = rmp_server->alloc(packet->npages, packet->size); + if (new_hndl == -1) + { + packet->error = ALLOC_FAILED; + return; + } + packet->hndl = new_hndl; +} + +void handle_read(rmp::Server *rmp_server, rmp::packet *packet) +{ + + rmp::handle hndl = packet->hndl; + if (!(rmp_server->contains_key(hndl))) + { + packet->error = INVALID_HANDLE; + return; + } + + int read_suc = rmp_server->read(hndl, packet); + if (read_suc == -1) + { + packet->error = READ_FAILED; + return; + } +} + +void handle_write(rmp::Server *rmp_server, rmp::packet *packet) +{ + + rmp::handle hndl = packet->hndl; + if (!(rmp_server->contains_key(hndl))) + { + packet->error = INVALID_HANDLE; + return; + } + + int write_suc = rmp_server->write(hndl, packet); + if (write_suc == -1) + { + packet->error = WRITE_FAILED; + return; + } +} + +void handle_free(rmp::Server *rmp_server, rmp::packet *packet) +{ + + rmp::handle hndl = packet->hndl; + if (!(rmp_server->contains_key(hndl))) + { + packet->error = INVALID_HANDLE; + return; + } + + int write_suc = rmp_server->free(hndl, packet->size); + if (write_suc == -1) + { + packet->error = FREE_FAILED; + return; + } +} + +void rmp::Server::handle(rmp::packet *packet) +{ + switch (packet->action) + { + case 0: + handle_alloc(this, packet); + break; + case 1: + handle_read(this, packet); + break; + case 2: + handle_write(this, packet); + break; + case 3: + handle_free(this, packet); + break; + default: + packet->error = UNKNOWN_ACTION; + break; + } +} + +rmp::handle rmp::Server::alloc(u32 npages, u32 page_size) +{ + ul mem_region_size = npages * page_size; + + void *remote_mem_region = mmap(NULL, mem_region_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + if (remote_mem_region == MAP_FAILED) + { + return -1; + } + + for (u32 i = 0; i < npages; i++) + { + } + u32 new_handle = this->get_new_handle(); + this->set_addr_map(new_handle, remote_mem_region); + this->set_hndl_n_pages(new_handle, npages); + return new_handle; +} + +int rmp::Server::read(rmp::handle handle, rmp::packet *packet) +{ + u32 offset = packet->offset; + u32 npages = + this->get_hndl_n_pages(handle); + ul addr = (ul)this->get_addr_map(handle); + if (offset >= npages) + { + return -1; + } + + ul addr_with_offset = addr + (packet->size * offset); + memcpy(packet->data, (void *)addr_with_offset, sizeof(packet->data)); + return 0; +} + +int rmp::Server::write(rmp::handle handle, rmp::packet *packet) +{ + u32 offset = packet->offset; + u32 npages = + this->get_hndl_n_pages(handle); + ul addr = (ul)this->get_addr_map(handle); + + if (offset >= npages) + { + return -1; + } + + ul addr_with_offset = addr + (packet->size * offset); + memcpy((void *)addr_with_offset, packet->data, sizeof(packet->data)); + return 0; +} + +int rmp::Server::free(rmp::handle handle, u32 page_size) +{ + if (this->contains_key(handle)) + { + u32 npages = + this->get_hndl_n_pages(handle); + if (npages == 0) + { + return -1; + } + + void *addr = this->get_addr_map(handle); + long size = npages * page_size; + if (munmap(addr, size) == -1) + { + return -1; + } + + this->unset_addr_map(handle); + this->unset_hndl_n_pages(handle); + return 0; + } + return -1; +} \ No newline at end of file diff --git a/server/rmp_test.cpp b/server/rmp_test.cpp new file mode 100644 index 0000000..84b98f0 --- /dev/null +++ b/server/rmp_test.cpp @@ -0,0 +1,416 @@ + +#include +#include +#include +#include "rmp_types.hpp" +#include "rmp_server.hpp" + +int passed = 0, failed = 0; + +bool test(bool test, bool set) +{ + return test && set; +} + +rmp::packet *create_new_packet() +{ + rmp::packet *p = new rmp::packet; + p->error = 0; + p->size = 0; + p->npages = 0; + p->offset = 0; + return p; +} + +rmp::packet *get_alloc_packet(u32 npages, u32 page_size) +{ + rmp::packet *p = create_new_packet(); + p->action = 0; + p->npages = npages; + p->size = page_size; + p->hndl = -1; + return p; +} + +rmp::packet *get_read_packet(rmp::handle hndl, u32 offset) +{ + rmp::packet *p = create_new_packet(); + p->action = 1; + p->hndl = hndl; + p->offset = offset; + return p; +} + +rmp::packet *get_write_packet(rmp::handle hndl, u32 offset, char *data) +{ + rmp::packet *p = create_new_packet(); + p->action = 2; + p->hndl = hndl; + p->offset = offset; + strncpy(p->data, data, 4096); + return p; +} + +rmp::packet *get_free_packet(rmp::handle hndl) +{ + rmp::packet *p = create_new_packet(); + p->action = 3; + p->hndl = hndl; + return p; +} + +// test alloc +bool test_alloc_basic() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *alloc_req = get_alloc_packet(10, 4096); + + s->handle(alloc_req); + + valid = test(alloc_req->error == 0, alloc_req->hndl == 0); + + delete s, alloc_req; + return valid; +} + +// test alloc +bool test_alloc_100pages() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *alloc_req = get_alloc_packet(100, 4096); + + s->handle(alloc_req); + + valid = test(alloc_req->error == 0, alloc_req->hndl == 0); + + delete s, alloc_req; + return valid; +} + +// test alloc +bool test_alloc_a_lot_of_pages() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *alloc_req = get_alloc_packet(1000000000000, 4096); + + s->handle(alloc_req); + + // std::cout << alloc_req->error << std::endl; + valid = test(alloc_req->error == ALLOC_FAILED, true); + + delete s, alloc_req; + return valid; +} + +// test read +bool test_read() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *alloc_req = new rmp::packet; + alloc_req->action = 0; + alloc_req->npages = 10; + alloc_req->size = 4096; + alloc_req->error = 0; + + s->handle(alloc_req); + + valid = test(alloc_req->hndl == 0, true); + + char *msg = "Hello World!"; + rmp::packet *write_req = new rmp::packet; + write_req->action = 2; + write_req->offset = 0; + write_req->size = 4096; + write_req->error = 0; + write_req->hndl = alloc_req->hndl; + strncpy(write_req->data, msg, strlen(msg)); + + s->handle(write_req); + + rmp::packet *read_req = new rmp::packet; + read_req->action = 1; + read_req->offset = 0; + read_req->size = 4096; + read_req->error = 0; + read_req->hndl = alloc_req->hndl; + + s->handle(read_req); + + valid = test(strcmp(read_req->data, msg) == 0, valid); + // std::cout << read_req->data << std::endl; + + delete s, alloc_req, write_req, read_req; + return valid; +} + +// test read +bool test_read_wrong_offset() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *alloc_req = new rmp::packet; + alloc_req->action = 0; + alloc_req->npages = 10; + alloc_req->size = 4096; + alloc_req->error = 0; + + s->handle(alloc_req); + + assert(alloc_req->hndl == 0); + valid = test(alloc_req->hndl == 0, true); + + char *msg = "Hello World!"; + rmp::packet *write_req = new rmp::packet; + write_req->action = 2; + write_req->offset = 0; + write_req->size = 4096; + write_req->error = 0; + write_req->hndl = alloc_req->hndl; + strncpy(write_req->data, msg, strlen(msg)); + + s->handle(write_req); + + rmp::packet *read_req = new rmp::packet; + read_req->action = 1; + read_req->offset = 11; + read_req->size = 4096; + read_req->error = 0; + read_req->hndl = alloc_req->hndl; + + s->handle(read_req); + + valid = test(read_req->error == READ_FAILED, valid); + delete s, alloc_req, write_req, read_req; + return valid; +} + +// test read +bool test_read_wrong_handle() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *read_req = new rmp::packet; + read_req->action = 1; + read_req->offset = 0; + read_req->size = 4096; + read_req->error = 0; + read_req->hndl = 0; + + s->handle(read_req); + + valid = test(read_req->error == INVALID_HANDLE, true); + + delete s, read_req; + return valid; +} + +// test write +bool test_write() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *alloc_req = new rmp::packet; + alloc_req->action = 0; + alloc_req->npages = 10; + alloc_req->size = 4096; + alloc_req->error = 0; + + s->handle(alloc_req); + + valid = test(alloc_req->hndl == 0, true); + + char *msg = "Hello World!"; + rmp::packet *write_req = new rmp::packet; + write_req->action = 2; + write_req->offset = 0; + write_req->size = 4096; + write_req->error = 0; + write_req->hndl = alloc_req->hndl; + strncpy(write_req->data, msg, strlen(msg)); + + s->handle(write_req); + + valid = test(write_req->error == 0, valid); + + delete s, alloc_req, write_req; + return valid; +} + +// test write +bool test_write_wrong_offset() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *alloc_req = new rmp::packet; + alloc_req->action = 0; + alloc_req->npages = 10; + alloc_req->size = 4096; + alloc_req->error = 0; + + s->handle(alloc_req); + + valid = test(alloc_req->hndl == 0, true); + + char *msg = "Hello World!"; + rmp::packet *write_req = new rmp::packet; + write_req->action = 2; + write_req->offset = 11; + write_req->size = 4096; + write_req->error = 0; + write_req->hndl = alloc_req->hndl; + strncpy(write_req->data, msg, strlen(msg)); + + s->handle(write_req); + + valid = test(write_req->error == WRITE_FAILED, valid); + + delete s, alloc_req, write_req; + return valid; +} + +// test write +bool test_write_wrong_handle() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + + char *msg = "Hello World!"; + rmp::packet *write_req = new rmp::packet; + write_req->action = 2; + write_req->offset = 0; + write_req->size = 4096; + write_req->error = 0; + write_req->hndl = 0; + strncpy(write_req->data, msg, strlen(msg)); + + s->handle(write_req); + + assert(write_req->error == INVALID_HANDLE); + valid = test(write_req->error == INVALID_HANDLE, true); + + delete s, write_req; + return valid; +} + +// test free +bool test_free() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *alloc_req = new rmp::packet; + alloc_req->action = 0; + alloc_req->npages = 10; + alloc_req->size = 4096; + alloc_req->error = 0; + + s->handle(alloc_req); + + // std::cout << alloc_req->error << std::endl; + valid = test(alloc_req->hndl == 0, true); + + rmp::packet *free_req = new rmp::packet; + free_req->action = 3; + free_req->size = 4096; + free_req->error = 0; + free_req->hndl = alloc_req->hndl; + + s->handle(free_req); + + // std::cout << free_req->error << std::endl; + valid = test(free_req->error == 0, valid); + + delete s, alloc_req, free_req; + return valid; +} + +// test free +bool test_free_and_read() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *alloc_req = new rmp::packet; + alloc_req->action = 0; + alloc_req->npages = 10; + alloc_req->size = 4096; + alloc_req->error = 0; + + s->handle(alloc_req); + + // std::cout << alloc_req->error << std::endl; + valid = test(alloc_req->hndl == 0, true); + + rmp::packet *free_req = new rmp::packet; + free_req->action = 3; + free_req->size = 4096; + free_req->error = 0; + free_req->hndl = alloc_req->hndl; + + s->handle(free_req); + + // std::cout << free_req->error << std::endl; + valid = test(free_req->error == 0, valid); + + rmp::packet *read_req = new rmp::packet; + read_req->action = 1; + read_req->offset = 0; + read_req->size = 4096; + read_req->error = 0; + read_req->hndl = alloc_req->hndl; + + s->handle(read_req); + + // std::cout << read_req->error << std::endl; + valid = test(read_req->error == INVALID_HANDLE, valid); + delete s, alloc_req, free_req, read_req; + return valid; +} + +bool test_free_wrong_handle() +{ + bool valid; + rmp::Server *s = new rmp::Server(rmp::config()); + rmp::packet *free_req = new rmp::packet; + free_req->action = 3; + free_req->size = 4096; + free_req->error = 0; + free_req->hndl = 0; + + s->handle(free_req); + + valid = test(free_req->error == INVALID_HANDLE, true); + + delete s, free_req; + return valid; +} + +void runner(bool func(void), std::string name) +{ + + if (func()) + { + std::cout << name << " PASSED +" << ++passed << " -" << failed << std::endl; + } + else + { + std::cout << name << " FAILED +" << passed << " -" << ++failed << std::endl; + } +} + +int main() +{ + runner(test_alloc_basic, "test_alloc_basic "); + runner(test_alloc_a_lot_of_pages, "test_alloc_a_lot_of_pages"); + runner(test_read, "test_read "); + runner(test_read_wrong_offset, "test_read_wrong_offset "); + runner(test_read_wrong_handle, "test_read_wrong_handle "); + runner(test_write, "test_write "); + runner(test_write_wrong_offset, "test_write_wrong_offset "); + runner(test_write_wrong_handle, "test_write_wrong_handle "); + runner(test_free, "test_free "); + runner(test_free, "test_free_and_read "); + runner(test_free_wrong_handle, "test_free_wrong_handle "); +} diff --git a/server/server.cpp b/server/server.cpp index 5d61880..2de271a 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -1,77 +1,250 @@ -// Server side C/C++ program to demonstrate Socket programming -#include -#include -#include -#include -#include -#include +// Server side C/C++ program to demonstrate Socket programming +#include +#include +#include +#include +#include +#include +#include #include -#include -#define PORT 8080 - -int create_mmap(size_t sz) { - int handle = 12341; - char* addr = (char*)mmap(NULL, sz, PROT_WRITE, MAP_PRIVATE, MAP_ANONYMOUS, 0); - hasbtable_insert(addr, handle); - printf("mmap address: %s\n", addr); - return fd; + +#include "rmp_server.hpp" + +#define PORT 6767 + +int server_fd, client_sock, *new_sock; +rmp::Server *server; + +void *connection_handler(void *); +void init() +{ + server = new rmp::Server(rmp::config()); } -int main(int argc, char const *argv[]) -{ - int server_fd, new_socket, valread; - struct sockaddr_in address; - int opt = 1; - int addrlen = sizeof(address); - char buffer[1024] = {0}; - const char *hello = "Hello from server"; - - // Creating socket file descriptor - if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) - { - perror("socket failed"); - exit(EXIT_FAILURE); - } - - // Forcefully attaching socket to the port 8080 - if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, - &opt, sizeof(opt))) - { - perror("setsockopt"); - exit(EXIT_FAILURE); - } - address.sin_family = AF_INET; - address.sin_addr.s_addr = INADDR_ANY; - address.sin_port = htons( PORT ); - - // Forcefully attaching socket to the port 8080 - if (bind(server_fd, (struct sockaddr *)&address, - sizeof(address))<0) - { - perror("bind failed"); - exit(EXIT_FAILURE); - } - if (listen(server_fd, 3) < 0) - { - perror("listen"); - exit(EXIT_FAILURE); - } - if ((new_socket = accept(server_fd, (struct sockaddr *)&address, - (socklen_t*)&addrlen))<0) - { - perror("accept"); - exit(EXIT_FAILURE); - } - - valread = read( new_socket, buffer, 1024); - int page_sz = atoi(buffer); - printf("Page size received: %d\n", page_sz); - - int fd = create_mmap(page_sz); - char fd_str[20]; - sprintf(fd_str, "%d", fd); - send(new_socket, fd_str, strlen(fd_str), 0); - printf("File descriptor sent: %s\n", fd_str); - close(fd); - return 0; -} +/* + * Reads n bytes from sd into where p points to. + * + * returns 0 on succes or -1 on error. + * + * Note: + * The function's name is inspired by and dedicated to "W. Richard Stevens" (RIP). + */ +int readn(int sd, void *p, size_t n) +{ + size_t bytes_to_read = n; + size_t bytes_read = 0; + + while (bytes_to_read > bytes_read) + { + ssize_t result = read(sd, p + bytes_read, bytes_to_read); + if (-1 == result) + { + if ((EAGAIN == errno) || (EWOULDBLOCK == errno)) + { + continue; + } + +#ifdef DEBUG + { + int errno_save = errno; + perror("read() failed"); + errno = errno_save; + } +#endif + + break; + } + else if (0 == result) + { +#ifdef DEBUG + { + int errno_save = errno; + fprintf(stderr, "%s: Connection closed by peer.", __FUNCTION__); + errno = errno_save; + } +#endif + + break; + } + + bytes_to_read -= result; + bytes_read += result; + } + + return (bytes_read < bytes_to_read) ? -1 : 0; +} + +int writen(const int sd, void *b, const size_t s, const int retry_on_interrupt) +{ + size_t n = s; + while (0 < n) + { + ssize_t result = write(sd, b, n); + if (-1 == result) + { + if ((retry_on_interrupt && (errno == EINTR)) || (errno == EWOULDBLOCK) || (errno == EAGAIN)) + { + continue; + } + else + { + break; + } + } + + n -= result; + b += result; + } + + return (0 < n) ? -1 : 0; +} + +int main(int argc, char const *argv[]) +{ + int opt = 1; + struct sockaddr_in address; + int addrlen = sizeof(address); + + if (argc != 3) + { + std::cout << "Invalid args!" << std::endl; + std::cout << "Usage: ./server " << std::endl; + return 1; + } + + std::cout << argv[1] << std::endl; + std::cout << atoi(argv[2]) << std::endl; + + init(); + + // Creating socket file descriptor + if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) + { + perror("socket failed"); + exit(EXIT_FAILURE); + } + + // Forcefully attaching socket to the port 8080 + if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, + &opt, sizeof(opt))) + { + perror("setsockopt"); + exit(EXIT_FAILURE); + } + address.sin_family = AF_INET; + address.sin_addr.s_addr = inet_addr(argv[1]); + address.sin_port = htons(atoi(argv[2])); + + // Forcefully attaching socket to the port 8080 + if (bind(server_fd, (struct sockaddr *)&address, + sizeof(address)) < 0) + { + perror("bind failed"); + exit(EXIT_FAILURE); + } + if (listen(server_fd, 3) < 0) + { + perror("listen"); + exit(EXIT_FAILURE); + } + + while ((client_sock = accept(server_fd, (struct sockaddr *)&address, (socklen_t *)&addrlen))) + { + puts("Connection accepted"); + + pthread_t sniffer_thread; + rmp::thread_req *thread_info = new rmp::thread_req; + thread_info->client_sock = client_sock; + thread_info->server = server; + + if (pthread_create(&sniffer_thread, NULL, connection_handler, (void *)thread_info) < 0) + { + perror("could not create thread"); + return 1; + } + + pthread_join(sniffer_thread, NULL); + puts("Thread assigned"); + } + + if (client_sock < 0) + { + perror("Client connection accept failed!"); + return 1; + } + + return 0; +} + +void *connection_handler(void *thread_info) +{ + int read_size; + rmp::packet *req = new rmp::packet; + rmp::thread_req *info = (rmp::thread_req *)thread_info; + int sock = info->client_sock; + + // while (readn(sock, req, sizeof(*req)) != -1) + while ((read_size = recv(sock, req, sizeof(*req), 0)) > 0) + { + // printf("Recieved Bytes : %ld\n", read_size); + // printf("action : %d\n", req->action); + // printf("handle : %d\n", req->hndl); + // printf("error : %d\n", req->error); + // if (read_size == sizeof(rmp::packet)) + // { + // if (req->action == 0 || req->action == 1 || req->action == 2 || req->action == 3) + // { + info->server->handle(req); + // printf("Handle Returned For Action : %d\n", req->action); + + size_t sent_bytes = send(sock, req, sizeof(rmp::packet), 0); + // sleep(0.1); + // printf("Sent Bytes : %d\n", sent_bytes); + + // int retry_on_interrupt = 1; + // int result = writen(sock, req, sizeof(*req), retry_on_interrupt); + // if (-1 == result) + // { + // perror("writen()"); + // } + + // printf("Finished writing back results : %d\n", req->action); + if (req->action == 0) + { + printf("Allocated %d pages with handle : %d\n", req->npages, req->hndl); + } + else if (req->action == 1) + { + printf("Client Reading page : %d for handle : %d\n", req->offset, req->hndl); + } + else if (req->action == 2) + { + printf("Client Writing page : %d for handle : %d\n", req->offset, req->hndl); + } + else if (req->action == 3) + { + printf("Client Freeing handle : %d\n", req->hndl); + } + // } + // } + // else + // { + // break; + // } + } + + printf("READ SIZE IN THE END : %d\n", read_size); + + if (read_size == 0) + { + puts("Client disconnected"); + fflush(stdout); + } + else if (read_size == -1) + { + perror("recv failed"); + } + + free(thread_info); + return 0; +} \ No newline at end of file diff --git a/server/server_out b/server/server_out new file mode 100644 index 0000000..eff4f56 --- /dev/null +++ b/server/server_out @@ -0,0 +1,2 @@ +10.237.15.93 +6768 diff --git a/temp b/temp new file mode 100644 index 0000000..87bc9e0 --- /dev/null +++ b/temp @@ -0,0 +1,19 @@ +--------------INT RAND 10240 --------------- +Write Time 82.4949 seconds. +Read Time 81.7159 seconds. +Total Time 164.211 seconds. +Avg Read Time 0.00798007 seconds. +Avg Write Time 0.00805615 seconds. +Avg Total Time 0.0160362 seconds. +Sum 97697967 +SUM is invalid + +--------------INT RAND 102400 --------------- +Write Time 1007.39 seconds. +Read Time 1002 seconds. +Total Time 2009.38 seconds. +Avg Read Time 0.00978514 seconds. +Avg Write Time 0.00983775 seconds. +Avg Total Time 0.0196229 seconds. +Sum 1795668907 +SUM is invalid diff --git a/uffdman.cpp b/uffdman.cpp deleted file mode 100644 index da69bba..0000000 --- a/uffdman.cpp +++ /dev/null @@ -1,244 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define errExit(msg) do { perror(msg); exit(EXIT_FAILURE); } while (0) - -/* - TODO: - - remove comments - - remove unnecessary prints logging - - convert C to C++ where ever possible -*/ - -static int page_size = -1; - -static unordered_map addr_tid_map; -static unordered_map addr_uffd_map; -static unordered_map uffd_addr_map; - -static void (*page_resolver) (char* start_addr, char* faulting_addr, long page_size, int is_write, char** page); - -static void* fault_handler_thread(void *arg) -{ - static struct uffd_msg msg; /* Data read from userfaultfd */ - static int fault_cnt = 0; /* Number of faults so far handled */ - long uffd; /* userfaultfd file descriptor */ - static char *page = NULL; - struct uffdio_copy uffdio_copy; - ssize_t nread; - - uffd = (long) arg; - - /* Create a page that will be copied into the faulting region */ -/* - if (page == NULL) { - page = mmap(NULL, page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (page == MAP_FAILED) - errExit("mmap"); - } -*/ - /* Loop, handling incoming events on the userfaultfd - file descriptor */ - - for (;;) { - - /* See what poll() tells us about the userfaultfd */ - - struct pollfd pollfd; - int nready; - pollfd.fd = uffd; - pollfd.events = POLLIN; - nready = poll(&pollfd, 1, -1); - if (nready == -1) - errExit("poll"); - - printf("\nfault_handler_thread():\n"); - printf(" poll() returns: nready = %d; " - "POLLIN = %d; POLLERR = %d\n", nready, - (pollfd.revents & POLLIN) != 0, - (pollfd.revents & POLLERR) != 0); - - /* Read an event from the userfaultfd */ - - nread = read(uffd, &msg, sizeof(msg)); - if (nread == 0) { - printf("EOF on userfaultfd!\n"); - exit(EXIT_FAILURE); - } - - if (nread == -1) - errExit("read"); - - /* We expect only one kind of event; verify that assumption */ - - if (msg.event != UFFD_EVENT_PAGEFAULT) { - fprintf(stderr, "Unexpected event on userfaultfd\n"); - exit(EXIT_FAILURE); - } - - /* Display info about the page-fault event */ - - printf(" UFFD_EVENT_PAGEFAULT event: "); - printf("flags = %llx; ", msg.arg.pagefault.flags); - printf("address = %llx\n", msg.arg.pagefault.address); - - /* Copy the page pointed to by 'page' into the faulting - region. Vary the contents that are copied in, so that it - is more obvious that each fault is handled separately. */ -/* - memset(page, 'A' + fault_cnt % 20, page_size); -*/ - fault_cnt++; - - char* region_start_addr = (char*) uffd_addr_map[uffd]; - - page_resolver(region_start_addr, msg.arg.pagefault.address, msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE, &page); - - /* - TODO How to handle pagefault due to a write? - - Example: - addr = mmap(...); // demand paged mapping - addr[offset] = some_data; - |-------| -> how to get this in page fault? - */ - - - uffdio_copy.src = (unsigned long) page; - - /* We need to handle page faults in units of pages(!). - So, round faulting address down to page boundary */ - - uffdio_copy.dst = (unsigned long) msg.arg.pagefault.address & - ~(page_size - 1); - uffdio_copy.len = page_size; - uffdio_copy.mode = 0; - uffdio_copy.copy = 0; - if (ioctl(uffd, UFFDIO_COPY, &uffdio_copy) == -1) - errExit("ioctl-UFFDIO_COPY"); - - printf(" (uffdio_copy.copy returned %lld)\n", - uffdio_copy.copy); - } -} - -void register_uffd_page_resolver(void (*resolver) (char* start_addr, char* faulting_addr, long page_size, int is_write, char** page)) { - page_resolver = resolver; -} - -void unregister_uffd_page_resolver() { - callback = NULL; -} - -void register_uffd_region(char *addr, unsigned long n_pages) -{ - long uffd; /* userfaultfd file descriptor */ -// char *addr; /* Start of region handled by userfaultfd */ - unsigned long len; /* Length of region handled by userfaultfd */ - pthread_t thr; /* ID of thread that handles page faults */ - struct uffdio_api uffdio_api; - struct uffdio_register uffdio_register; - pthread_t thrd_id; -/* - if (argc != 2) { - fprintf(stderr, "Usage: %s num-pages\n", argv[0]); - exit(EXIT_FAILURE); - } -*/ - if(page_size==-1) page_size = sysconf(_SC_PAGE_SIZE); - - len = n_pages * page_size; - - /* Create and enable userfaultfd object */ - - uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); - if (uffd == -1) - errExit("userfaultfd"); - - uffdio_api.api = UFFD_API; - uffdio_api.features = 0; - if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) - errExit("ioctl-UFFDIO_API"); - - /* Create a private anonymous mapping. The memory will be - demand-zero paged--that is, not yet allocated. When we - actually touch the memory, it will be allocated via - the userfaultfd. */ -/* - addr = mmap(NULL, len, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (addr == MAP_FAILED) - errExit("mmap"); - - printf("Address returned by mmap() = %p\n", addr); -*/ - /* Register the memory range of the mapping we just created for - handling by the userfaultfd object. In mode, we request to track - missing pages (i.e., pages that have not yet been faulted in). */ - - uffdio_register.range.start = (unsigned long) addr; - uffdio_register.range.len = len; - uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; - if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) - errExit("ioctl-UFFDIO_REGISTER"); - - /* Create a thread that will process the userfaultfd events */ - - thrd_id = pthread_create(&thr, NULL, fault_handler_thread, (void *) uffd); - if (thrd_id != 0) { - errno = thrd_id; - errExit("pthread_create"); - } - - addr_tid_map[(unsigned long) addr] = thrd_id; - addr_uffd_map[(unsigned long) addr] = uffd; - uffd_addr_map[uffd] = (unsigned long) addr; - /* Main thread now touches memory in the mapping, touching - locations 1024 bytes apart. This will trigger userfaultfd - events for all pages in the region. */ -/* - int l; - l = 0xf; -*/ - /* Ensure that faulting address is not on a page - boundary, in order to test that we correctly - handle that case in fault_handling_thread() */ -/* - while (l < len) { - char c = addr[l]; - printf("Read address %p in main(): ", addr + l); - printf("%c\n", c); - l += 1024; - usleep(100000); // Slow things down a little - } - - exit(EXIT_SUCCESS); -*/ -} - -void unregister_uffd_regoin(char* addr) { - unsigned long ul_addr = (unsigned long) addr; - pthread_t tid = addr_tid_map[ul_addr]; - pthread_cancel(tid); - addr_tid_map.erase(tid); - long uffd = addr_uffd_map[ul_addr]; - addr_uffd_map.erase(ul_addr); - uffd_addr_map.erase(uffd); - -} diff --git a/uffdman.hpp b/uffdman.hpp deleted file mode 100644 index bdf7c85..0000000 --- a/uffdman.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef UFFDMAN_H -#define UFFDMAN_H - -void register_uffd_page_resolver(void (*handler) (char* start_addr, char* faulting_addr, long page_size, int is_write, char** page)); - -void unregister_uffd_page_resolver(); - -void register_uffd_region(char *addr, unsigned long n_pages); - -void unregister_uffd_regoin(char* addr); - -#endif diff --git a/uffdman/uffdman.cpp b/uffdman/uffdman.cpp new file mode 100644 index 0000000..8211506 --- /dev/null +++ b/uffdman/uffdman.cpp @@ -0,0 +1,383 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "uffdman.hpp" + +#ifndef DEBUG_PRINTS +#define DEBUG_PRINTS 0 +#endif + +// #if DEBUG_PRINTS +// #define err(msg) perror(msg) +// #define cout(exp...) printf(exp); +// #else +// #define err(msg) +// #define cout(exp...) +// #endif + +#if DEBUG_PRINTS +#define err(msg) perror(msg) +#define print(exp...) printf(exp); +#else +#define err(msg) +#define print(exp...) +#endif + +#define PAGE_SIZE sysconf(_SC_PAGE_SIZE) +#define PGROUNDDOWN(a) (((a)) & ~(PAGE_SIZE - 1)) + +using namespace std; + +bool uffd_initialized = false; + +mutex fault_handler_cr; + +static unordered_map *addr_tid_map; + +// Double sided maps for starting address of a region to uffd +static unordered_map *start_addr_uffd_map; +static unordered_map *uffd_start_addr_map; + +// Maps to maintain prev resolved pages for lazy page resolving +static unordered_map *uffd_prev_resolved_addr; +static unordered_map *uffd_prev_resolved_op; + +static void (*page_resolver)(void *start_addr, void *faulting_addr, int is_write, void *page); + +void uffdman_init() +{ + + if (!uffd_initialized) + { + uffd_prev_resolved_addr = new unordered_map(); + uffd_prev_resolved_op = new unordered_map(); + addr_tid_map = new unordered_map(); + start_addr_uffd_map = new unordered_map(); + uffd_start_addr_map = new unordered_map(); + uffd_initialized = true; + } +} + +void uffdman_destroy() +{ + delete addr_tid_map; + delete start_addr_uffd_map; + delete uffd_start_addr_map; + delete uffd_prev_resolved_addr; + delete uffd_prev_resolved_op; +} + +/* + + Off-by-one Invalidation: If a pagefault occurs, then the faulting address and operation (read/write) + is saved, so that it can be invalidated when the next pagefault occurs. + + This is useful because a new pagefault means access to new address, so we can invalidate the previous + one and if the recently invalidated address is accessed then that will cause a pagefault which invalidates + the one resolved before and so on. + + This is done to solve two problems: + 1. A pagefault (userfault) won't occur for subsquent reads/writes once it is resolved/handled. + 2. In a pagefault due to write we won't get the data (part of faulting instruction) that is + being written in the userfault handler. + + Constraints: + 1. Atmost two pages of memory is required. + 2. The memory of entity which is resoving pages will be behind by one memory address in terms of + consistency. Since the name Off-by-one. + +*/ + +// template +// // template +// static bool exists(std::unordered_map &m, Key key) +// { +// auto got = m.find(key); + +// if (got == m.end()) +// return false; +// return true; +// } + +static void invalidate_prev_resolved_page(long uffd) +{ + auto prev_resolved_addr_exists = uffd_prev_resolved_addr->find(uffd); + auto prev_resolved_op_exists = uffd_prev_resolved_op->find(uffd); + auto region_start_addr_exists = uffd_start_addr_map->find(uffd); + + if (prev_resolved_addr_exists != uffd_prev_resolved_addr->end() && + prev_resolved_op_exists != uffd_prev_resolved_op->end() && + region_start_addr_exists != uffd_start_addr_map->end()) + { + + unsigned long prev_resolved_addr = uffd_prev_resolved_addr->at(uffd); + int prev_resolved_op = uffd_prev_resolved_op->at(uffd); + void *region_start_addr = (void *)uffd_start_addr_map->at(uffd); + + // Invalidate previous page after resolving it + void *prev_page = (void *)prev_resolved_addr; + + print("Prev Resolved Address = %" PRIx64 "; ", prev_resolved_addr); + + if (prev_resolved_op) + { + // Resolve only if write + print("invalidate_prev_resolved_page : Saving page to server as the page is dirty\n"); + page_resolver(region_start_addr, (void *)prev_resolved_addr, prev_resolved_op, prev_page); + } + + madvise((void *)PGROUNDDOWN(prev_resolved_addr), PAGE_SIZE, MADV_DONTNEED); + } +} + +static void handle_unmap_event(long uffd) +{ + void *region_start_addr = (void *)uffd_start_addr_map->at(uffd); + uffdman_unregister_region(region_start_addr); +} + +static void *fault_handler_thread(void *arg) +{ + print("FAULT HANDLER STARTED\n"); + static struct uffd_msg msg; /* Data read from userfaultfd */ + static int fault_cnt = 0; /* Number of faults so far handled */ + long uffd; /* userfaultfd file descriptor */ + static void *page = NULL; + struct uffdio_copy uffdio_copy; + ssize_t nread; + + uffd = (long)arg; + + /* Create a page that will be copied into the faulting region */ + + if (page == NULL) + { + page = (void *)mmap(NULL, PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (page == MAP_FAILED) + print("mmap in userfault handler failed\n"); + } + + /* Loop, handling incoming events on the userfaultfd + file descriptor */ + + while (true) + { + /* See what poll() tells us about the userfaultfd */ + + struct pollfd pollfd; + int nready; + pollfd.fd = uffd; + pollfd.events = POLLIN; + nready = poll(&pollfd, 1, -1); + if (nready == -1) + break; + print("\nfault_handler_thread():\n"); + print(" poll() returns: nready = %d; POLLIN = %d; POLLERR = \n", nready, (int)((pollfd.revents & POLLIN) != 0), (int)((pollfd.revents & POLLERR) != 0)); + /* Read an event from the userfaultfd */ + + nread = read(uffd, &msg, sizeof(msg)); + if (nread == 0) + { + print("EOF on userfaultfd!\n"); + break; + } + + if (nread == -1) + break; + + if (msg.event == UFFD_EVENT_UNMAP) + { + // TODO: Handle arbitrary unmap event + continue; + } + /* We expect only one kind of event; verify that assumption */ + + if (msg.event != UFFD_EVENT_PAGEFAULT) + { + err("Unexpected event on userfaultfd"); + break; + } + + /* Display info about the page-fault event */ + print(" UFFD_EVENT_PAGEFAULT event: "); + print("uffd = %d; ", uffd); + print("flags = %" PRIx64 "; ", msg.arg.pagefault.flags); + print("address = %" PRIx64 "; ", msg.arg.pagefault.address); + print("Rounded address = %" PRIx64 "\n", PGROUNDDOWN(msg.arg.pagefault.address)); + fault_cnt++; + + void *faulting_addr = (void *)msg.arg.pagefault.address; + int faulting_op = msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE; + + std::lock_guard guard(fault_handler_cr); + + invalidate_prev_resolved_page(uffd); + + if (!faulting_op) + { // Resolve current pauge fault only if read + print("Calling rmp_page_resolver with faulting_addr : %" PRIx64 " and Rounded : %" PRIx64 "\n", faulting_addr, PGROUNDDOWN((unsigned long)faulting_addr)); + void *region_start_addr = (void *)uffd_start_addr_map->at(uffd); + page_resolver(region_start_addr, faulting_addr, faulting_op, page); + } + + /* + How to handle pagefault due to a write? + + Example: + addr = mmap(...); // demand paged mapping + addr[offset] = some_data; + |-------| -> how to get this in page fault? + + Solution: Off-by-one Invalidation. Meaning, handle the write during next pagefault. If no page + fault occurs after resolving a write then invalidate during unregistration. + + */ + + /* Copy the page pointed to by 'page' into the faulting + region. Vary the contents that are copied in, so that it + is more obvious that each fault is handled separately. */ + uffdio_copy.src = (unsigned long)page; + + /* We need to handle page faults in units of pages(!). + So, round faulting address down to page boundary */ + + uffdio_copy.dst = PGROUNDDOWN((unsigned long)faulting_addr); + uffdio_copy.len = PAGE_SIZE; + uffdio_copy.mode = 0; + uffdio_copy.copy = 0; + if (ioctl(uffd, UFFDIO_COPY, &uffdio_copy) == -1) + break; + print("\t(uffdio_copy.copy returned %lld\n", uffdio_copy.copy); + + // (*uffd_prev_resolved_addr)[uffd] = PGROUNDDOWN((unsigned long)faulting_addr); + // Check and Insert + auto prev_resolved_addr_it = uffd_prev_resolved_addr->find(uffd); + if (prev_resolved_addr_it != uffd_prev_resolved_addr->end()) + { + prev_resolved_addr_it->second = PGROUNDDOWN((unsigned long)faulting_addr); + } + else + { + uffd_prev_resolved_addr->insert({uffd, PGROUNDDOWN((unsigned long)faulting_addr)}); + } + + // Check and Insert + auto prev_resolved_op_it = uffd_prev_resolved_op->find(uffd); + if (prev_resolved_op_it != uffd_prev_resolved_op->end()) + { + prev_resolved_op_it->second = faulting_op; + } + else + { + uffd_prev_resolved_op->insert({uffd, faulting_op}); + } + } + + return 0; +} + +void uffdman_register_page_resolver(void (*resolver)(void *start_addr, void *faulting_addr, int is_write, void *page)) +{ + page_resolver = resolver; +} + +void uffdman_unregister_page_resolver() +{ + page_resolver = NULL; +} + +int uffdman_register_region(void *addr, unsigned long n_pages) +{ + long uffd; /* userfaultfd file descriptor */ + unsigned long len; /* Length of region handled by userfaultfd */ + struct uffdio_api uffdio_api; + struct uffdio_register uffdio_register; + pthread_t thrd_id; + + len = n_pages * PAGE_SIZE; + + /* Create and enable userfaultfd object */ + + uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); + if (uffd == -1) + { + err("Error in userfaultfd syscall"); + return -1; + } + + uffdio_api.api = UFFD_API; + uffdio_api.features = 0; + if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) + { + err("Error in ioctl UFFDIO_API"); + return -1; + } + + uffdio_register.range.start = (unsigned long)addr; + uffdio_register.range.len = len; + uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) + { + err("Error in ioctl UFFDIO_REGISTER"); + return -1; + } + + /* Create a thread that will process the userfaultfd events */ + int out = pthread_create(&thrd_id, NULL, fault_handler_thread, (void *)uffd); + if (out != 0) + { + errno = thrd_id; + err("Error while creating fault handler thread"); + return -1; + } + + addr_tid_map->insert({(unsigned long)addr, thrd_id}); + start_addr_uffd_map->insert({(unsigned long)addr, uffd}); + uffd_start_addr_map->insert({uffd, (unsigned long)addr}); + return 0; +} + +void uffdman_unregister_region(void *addr) +{ + unsigned long ul_addr = (unsigned long)addr; + auto start_addr_uffd_exists = start_addr_uffd_map->find(ul_addr); + if (start_addr_uffd_exists != start_addr_uffd_map->end()) + { + long uffd = start_addr_uffd_map->at(ul_addr); + invalidate_prev_resolved_page(uffd); + + start_addr_uffd_map->erase(ul_addr); + uffd_start_addr_map->erase(uffd); + + uffd_prev_resolved_addr->erase(uffd); + uffd_prev_resolved_op->erase(uffd); + + auto addr_tid_exists = addr_tid_map->find(ul_addr); + if (addr_tid_exists != addr_tid_map->end()) + { + pthread_t tid = addr_tid_map->at(ul_addr); + addr_tid_map->erase(tid); + pthread_cancel(tid); + } + } +}