Commit 993050da authored by Ciarán Ó Rourke's avatar Ciarán Ó Rourke
Browse files

Squashed 'external/fiphoboserver/' changes from fb9d2228..a9f73a85

a9f73a85 Merge branch 'devel' of git.ichec.ie:oilgas/ltfs/fiphoboserver into devel
64d9fe72 fix CmakeLists
35d03487 Merge branch 'refactor' into 'devel'
08fdc907 update CMake Lists to do all that was formerly done by the superbuild - which really is not that much
9a8740af Modularise project and implement abstract interfaces
98ca3603 Added .gitignore
9a00f70e Merge branch 'non_block_fifo' into 'devel'
2836db9f Fix for FIFO related race conditions
6bb805d2 Refactor source code to separate server, fifo, and backend logic
dcb20739 Split put and get backend functions into separate files
741edde0 Removed Phobos path variable that makes more sense in superfiphoboserver
7b80f609 Add FIFO name to debug print statements
f506f954 Update phobos include directory to allow for command line specification of fiphoboserver directory
5ab9d1d9 class definition needs ;
a61aef3d Ran Clang format
2a3cb78f Blocking files obsolete now
af247189 - Use randomly name fifo - getDataFromFIFO no longer uses its own thread - cleaned up some clutter
9ff8651d GET works
899241f9 Use download_file again
bd5aa1e0 Temporarily say this is ok
dd3011d8 Retry download_file
dd776876 Merge branch 'master' of git.ichec.ie:oilgas/ltfs/fiphoboserver
949005a8 Try to get size from Phobos
182c508d Add a request handler for unsuppoorted requests
7cd0839a Fixed typos
473b03e8 Fixed typos
801b7295 Further additions to get request
52ee3f48 Remove finish get for now
a8846b7e Fixed typos
feb589e4 Renamed StaticHandler class to GetRequestHandler
f459c890 Reflect renaming to get_request_handler
08002085 Fixed typo
19cb3ab6 Command line parsing of relevant variables
0307fae8 Attempts at get request
99e5e4b6 Ran clang format
f86198f0 Ran clang format
9114863f Ran clang format
29bfd8de Ran clang format
c938649e Clean up static handler, remove body output from push handler and make fiphobo compile again
df959ea4 Clean up a bit and add Exceptions
7c547c4a Can supply filename as command line argument
8d8a71d0 Merge branch 'master' of git.ichec.ie:oilgas/ltfs/fiphoboserver
23af509d Removed old debugging print functions
b8d04f51 Get rid of seg fault bug
1bb24039 Resolve merge conflict
139959fc Merge branch 'master' of git.ichec.ie:oilgas/ltfs/fiphoboserver
a9a31319 Probably cleaned up a merge conflict, not sure
6523f140 Use std::cout
cbfb6438 Divide the src directory for cmake into a library and an executable.
279c2d65 More c_str()s
0001a0c4 open needs c_str()
4bfe395a Takes key as command line argument
241b87be Accept key as command line argument
4a6a40b5 Removed unneccesary .str() calls
8fabfff8 Switch to non-blocking fiphobo
5966b57c BUILD_BINARIES variable no longer required

git-subtree-dir: external/fiphoboserver
git-subtree-split: a9f73a85
parent fb9d2228
*.swp
todo.txt
cmake_minimum_required(VERSION 3.0)
project(fiphoboserver)
get_filename_component(PHOBOS_INCLUDE_DIRECTORY ../phobos/src/include ABSOLUTE
${CMAKE_CURRENT_SOURCE_DIR})
set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
find_package(proxygen REQUIRED)
add_subdirectory(src)
......@@ -3,11 +3,11 @@
Build with
```
cmake -DFIPHOBOSERVER_BUILD_BINARIES=ON -DDEBUG=ON -DCMAKE_PREFIX_PATH=/path/to/proxygen /path/to/CMakeLists/file/
cmake -DDEBUG=ON -DCMAKE_PREFIX_PATH=/path/to/proxygen /path/to/CMakeLists/file/
make
```
from any directory you want and hope for the best! ;)
Oh, make sure Phobos include directories are under `../phobos/src/include` from the main `CMakeLists.txt`, or change that path in there
\ No newline at end of file
Oh, make sure Phobos include directories are under `../phobos/src/include` from the main `CMakeLists.txt`, or change that path in there
include_directories(${PHOBOS_INCLUDE_DIRECTORY})
include_directories(/usr/include/glib-2.0)
include_directories(/usr/lib64/glib-2.0/include)
if(DEBUG)
add_compile_definitions(DEBUG)
endif(DEBUG)
add_executable(
fiphoboserver
server/server.cc
server/static_handler.cc
server/push_request_handler.cc
fiphobo/fiphobo_blocking.cc
)
if(CUSTOM_OUTPUT_DIRECTORY)
set_target_properties( fiphoboserver
PROPERTIES
ARCHIVE_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver"
LIBRARY_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver"
RUNTIME_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver"
)
endif(CUSTOM_OUTPUT_DIRECTORY)
add_subdirectory(fiphobo/phobos_cpp_wrapper)
target_compile_features(fiphoboserver PUBLIC cxx_std_14)
target_link_libraries(fiphoboserver PUBLIC proxygen::proxygen)
target_link_libraries(fiphoboserver PUBLIC proxygen::proxygenhttpserver)
target_link_libraries(fiphoboserver PUBLIC phobos_store)
target_link_libraries(fiphoboserver PUBLIC phobos_cpp_wrapper)
add_subdirectory(storage)
add_subdirectory(stream)
add_subdirectory(server)
......@@ -2,81 +2,76 @@
#include <proxygen/lib/http/HTTPMessage.h>
namespace fiphoboserver
{
class S3_header
{
public:
namespace fiphoboserver {
class S3_header {
public:
void setHeaders(std::unique_ptr<proxygen::HTTPMessage> newHeaders)
{
headers = std::move(newHeaders);
}
std::string getBucket()
{
if(!headers)
{
if (!headers) {
return "";
}
std::string path = headers->getPath();
path = path.substr(1, path.size());
std::string path = headers->getPath();
path = path.substr(1, path.size());
std::string bucketName = path.substr(0, path.find('/'));
return bucketName;
}
std::string getKey()
{
if(!headers)
{
if (!headers) {
return "";
}
std::string path = headers->getPath();
path = path.substr(1, path.size());
std::string path = headers->getPath();
path = path.substr(1, path.size());
std::string fileKey = path.substr(path.find('/') + 1, path.size() - 1);
if(fileKey.find('/') != std::string::npos ||
fileKey.find('\\') != std::string::npos)
{
if (fileKey.find('/') != std::string::npos
|| fileKey.find('\\') != std::string::npos) {
std::cout << "Bad Key! " << std::endl;
return "";
}
return fileKey;
}
bool isCreateBucketRequest()
{
if(!headers)
{
if (!headers) {
return false;
}
if(getKey() == "")
return true;
if (getKey() == "") return true;
return false;
}
size_t getBodyLength()
{
std::string contentLength = headers->getHeaders().getSingleOrEmpty("Content-Length");
std::string contentLength =
headers->getHeaders().getSingleOrEmpty("Content-Length");
return static_cast<size_t>(std::stoi(contentLength));
}
std::vector<std::pair<std::string, std::string>> getMetaData()
{
std::vector<std::pair<std::string, std::string>> metadata;
std::string metaSearchString = "x-amz-meta-";
headers->getHeaders().forEach([&] (const std::string& header, const std::string& val) {
if(header.find(metaSearchString) != std::string::npos)
{
std::string metaName = header.substr(metaSearchString.size(), header.size() -1);
headers->getHeaders().forEach([&](const std::string& header,
const std::string& val) {
if (header.find(metaSearchString) != std::string::npos) {
std::string metaName =
header.substr(metaSearchString.size(), header.size() - 1);
std::pair<std::string, std::string> currentPair(metaName, val);
metadata.push_back(currentPair);
}
});
});
return metadata;
}
private:
private:
std::unique_ptr<proxygen::HTTPMessage> headers;
};
}
\ No newline at end of file
} // namespace fiphoboserver
#include "fiphobo.h"
#include <iostream>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <stdio.h>
#include <cstdio>
#include <stdlib.h>
#include <unistd.h>
namespace fiphoboserver
{
FiPhobo::FiPhobo()
{
memset(&descriptor, 0, sizeof(descriptor));
descriptor.xd_attrs = {0};
}
void FiPhobo::setBucketName(std::string bucketName)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::setBucketName" << std::endl;
#endif
int rc = pho_attr_set(&descriptor.xd_attrs, "bucket", bucketName.c_str());
if(rc)
{
printf("pho_attr_set returned an error: %d\n", rc);
return;
}
}
void FiPhobo::setMetaData(std::vector<std::pair<std::string, std::string>> metaData)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::setMetaData" << std::endl;
#endif
std::for_each(metaData.begin(), metaData.end(),
[&](std::pair<std::string, std::string> pair){
int rc = pho_attr_set(&descriptor.xd_attrs, pair.first.c_str(),
pair.second.c_str());
if(rc)
{
printf("pho_attr_set returned an error: %d\n", rc);
return;
}
});
}
void FiPhobo::startPutOperation(unsigned int size, std::string objectID)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::startPutOperation" << std::endl;
#endif
startFIFO();
char *unconstedObjectID = new char[objectID.length() + 1];
strcpy(unconstedObjectID, objectID.c_str());
descriptor.xd_op = PHO_XFER_OP_PUT;
descriptor.xd_objid = unconstedObjectID;
descriptor.xd_fd = fifo_descriptor;
descriptor.xd_size = size;
#ifdef DEBUG
std::cout << "Starting async phobos_put_cpp" << std::endl;
#endif
phobos_result = std::async([&](){
return phobos_put_cpp(&descriptor, 1, NULL, NULL);
});
//TODOOOO
// delete [] unconstedObjectID;
}
void FiPhobo::addDataToFIFO(std::unique_ptr<folly::IOBuf> buffer)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::addDataToFIFO" << std::endl;
#endif
if(ioRunning)
{
int ioRC = io_result.get();
if(ioRC < 0)
{
//TODO error handling
}
}
ioRunning = true;
buffer->coalesce();
io_result = std::async([&](){
return write(fifo_descriptor, buffer->data(), buffer->length());
});
}
void FiPhobo::startGetOperation(std::string& objectID)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::startGetOperation" << std::endl;
#endif
startFIFO();
char* unconstedObjectID = (char*) objectID.c_str();
descriptor.xd_op = PHO_XFER_OP_GET;
descriptor.xd_objid = unconstedObjectID;
descriptor.xd_fd = fifo_descriptor;
phobos_result = std::async([&](){
return phobos_get_cpp(&descriptor, 1, NULL, NULL);
});
}
std::unique_ptr<folly::IOBuf> FiPhobo::getDataFromFIFO(size_t size)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::getDataFromFIFO" << std::endl;
#endif
std::unique_ptr<folly::IOBuf> buffer = folly::IOBuf::create(size);
if(ioRunning)
{
int ioRC = io_result.get();
if(ioRC < 0)
{
//TODO error handling
}
}
ioRunning = true;
io_result = std::async([&](){
buffer->unshare();
return read(fifo_descriptor, buffer->writableData(), size);
});
return std::move(buffer);
}
void FiPhobo::finishIO()
{
#ifdef DEBUG
std::cout << "Phobos_Layer::finishIO" << std::endl;
#endif
int ioRC = io_result.get();
if(ioRC < 0)
{
//TODO error handling
}
int phobosRC = phobos_result.get();
if(phobosRC)
{
//TODO error handling!
}
ioRunning = false;
}
FiPhobo::~FiPhobo()
{
#ifdef DEBUG
std::cout << "Phobos_Layer::~Phobos_Layer" << std::endl;
#endif
// pho_attrs_free(&descriptor.xd_attrs); // TODO: Is that done in desc_destroy??
pho_xfer_desc_destroy_cpp(&descriptor);
if(fifo_descriptor > 0)
{
close(fifo_descriptor);
}
if( access( fifoName.c_str(), F_OK ) != -1 )
{
remove(fifoName.c_str());
}
}
void FiPhobo::startFIFO()
{
#ifdef DEBUG
std::cout << "Phobos_Layer::startFIFO" << std::endl;
#endif
int rc = mkfifo(fifoName.c_str(), 0777);
if(rc)
{
printf("mkfifo returned an error: %s\n", std::strerror(errno));
return;
}
fifo_descriptor = open(fifoName.c_str(), O_RDWR);
if (fifo_descriptor < 0)
{
printf("Could not open file: %s\n", fifoName);
return;
}
#ifdef DEBUG
std::cout << "End of Phobos_Layer::startFIFO" << std::endl;
#endif
}
} // namespace fiphoboserver
\ No newline at end of file
#pragma once
#include <future>
#include <vector>
#include <folly/io/IOBuf.h>
extern "C" {
#include "phobos_cpp_wrapper/phobos_cpp_wrapper.h"
}
namespace fiphoboserver {
// TODO: Remember the m-versions: PUT and GET functions can work on more than one
// file. Then this structure is stupid. Maybe with a vector of descriptors?
class FiPhobo
{
public:
FiPhobo();
void setBucketName(std::string bucketName);
void setMetaData(std::vector<std::pair<std::string, std::string>> metaData);
void startPutOperation(unsigned int size, std::string objectID);
void addDataToFIFO(std::unique_ptr<folly::IOBuf> buffer);
void startGetOperation(std::string& objectID);
std::unique_ptr<folly::IOBuf> getDataFromFIFO(size_t size);
void finishIO();
~FiPhobo();
private:
void startFIFO();
struct pho_xfer_desc descriptor = {0};
std::string fifoName = "/tmp/fiphoboserver_fifotemp";
int fifo_descriptor = -1;
std::future<int> phobos_result;
std::future<long int> io_result;
bool ioRunning = false;
};
} // namespace fiphoboserver
\ No newline at end of file
#include "fiphobo_blocking.h"
#include <iostream>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <stdio.h>
#include <cstdio>
#include <stdlib.h>
#include <unistd.h>
namespace fiphoboserver
{
FiPhobo::FiPhobo()
{
memset(&descriptor, 0, sizeof(descriptor));
descriptor.xd_attrs = {0};
}
void FiPhobo::setBucketName(std::string bucketName)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::setBucketName" << std::endl;
#endif
int rc = pho_attr_set(&descriptor.xd_attrs, "bucket", bucketName.c_str());
if(rc)
{
printf("pho_attr_set returned an error: %d\n", rc);
return;
}
}
void FiPhobo::setMetaData(std::vector<std::pair<std::string, std::string>> metaData)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::setMetaData" << std::endl;
#endif
std::for_each(metaData.begin(), metaData.end(),
[&](std::pair<std::string, std::string> pair){
int rc = pho_attr_set(&descriptor.xd_attrs, pair.first.c_str(),
pair.second.c_str());
if(rc)
{
printf("pho_attr_set returned an error: %d\n", rc);
return;
}
});
}
void FiPhobo::startPutOperation(int size, std::string objectID)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::startPutOperation" << std::endl;
#endif
startFIFO();
char *unconstedObjectID = new char[objectID.length() + 1];
strcpy(unconstedObjectID, objectID.c_str());
descriptor.xd_op = PHO_XFER_OP_PUT;
descriptor.xd_objid = unconstedObjectID;
descriptor.xd_fd = fifo_descriptor;
descriptor.xd_size = size;
// #ifdef DEBUG
// std::cout << "Starting async phobos_put_cpp" << std::endl;
// #endif
// phobos_result = std::async([&](){
// return phobos_put_cpp(&descriptor, 1, NULL, NULL);
// });
//
//TODOOOO
// delete [] unconstedObjectID;
}
void FiPhobo::addDataToFIFO(std::unique_ptr<folly::IOBuf> buffer)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::addDataToFIFO" << std::endl;
#endif
// if(ioRunning)
// {
// int ioRC = io_result.get();
// if(ioRC < 0)
// {
// //TODO error handling
// }
// }
// ioRunning = true;
//
// buffer->coalesce();
// io_result = std::async([&](){
// return write(fifo_descriptor, buffer->data(), buffer->length());
// });
size_t written = write(fifo_descriptor, buffer->data(), buffer->length());
}
void FiPhobo::startGetOperation(std::string& objectID)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::startGetOperation" << std::endl;
#endif
startFIFO();
char *unconstedObjectID = new char[objectID.length() + 1];
strcpy(unconstedObjectID, objectID.c_str());
descriptor.xd_op = PHO_XFER_OP_GET;
descriptor.xd_objid = unconstedObjectID;
descriptor.xd_fd = fifo_descriptor;
// phobos_result = std::async([&](){
// return phobos_get_cpp(&descriptor, 1, NULL, NULL);
// });
}
std::unique_ptr<folly::IOBuf> FiPhobo::getDataFromFIFO(size_t size)
{
#ifdef DEBUG
std::cout << "Phobos_Layer::getDataFromFIFO" << std::endl;
#endif
std::unique_ptr<folly::IOBuf> buffer = folly::IOBuf::create(size);
// if(ioRunning)
// {
// int ioRC = io_result.get();
// if(ioRC < 0)
// {
// //TODO error handling
// }
// }
// ioRunning = true;
//
// io_result = std::async([&](){
// buffer->unshare();
// return read(fifo_descriptor, buffer->writableData(), size);
// });
int phobos_return = phobos_get_cpp(&descriptor, 1, NULL, NULL);
if(phobos_return < 0)
{
std::cout << "GET: schade schade schade " << std::endl;
//error handling
}
lseek(fifo_descriptor, 0, SEEK_SET);
buffer->unshare();
read(fifo_descriptor, buffer->writableData(), size);
return std::move(buffer);
}
void FiPhobo::finishIO()
{
#ifdef DEBUG
std::cout << "Phobos_Layer::finishIO" << std::endl;
#endif
// int ioRC = io_result.get();
// if(ioRC < 0)
// {
// //TODO error handling
// }
//
// int phobosRC = phobos_result.get();
// if(phobosRC)
// {