Commit a02ea510 authored by Ciarán Ó Rourke's avatar Ciarán Ó Rourke
Browse files

Prepare fiphoboserver pull

parent bf66169c
---
Language: Cpp
AccessModifierOffset: -2
AlignAfterOpenBracket: AlwaysBreak
AlignConsecutiveAssignments: true
AlignConsecutiveDeclarations: false
AlignEscapedNewlines: Right
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: Yes
BinPackArguments: true
BinPackParameters: false
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
AfterExternBlock: false
BeforeCatch: true
BeforeElse: true
IndentBraces: false
SplitEmptyFunction: true
SplitEmptyRecord: true
SplitEmptyNamespace: true
BreakBeforeBinaryOperators: NonAssignment
BreakBeforeBraces: Stroustrup
BreakBeforeInheritanceComma: false
BreakInheritanceList: AfterColon
BreakBeforeTernaryOperators: false
BreakConstructorInitializersBeforeComma: false
BreakConstructorInitializers: AfterColon
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: false
ColumnLimit: 80
CommentPragmas: '^ *@'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
ForEachMacros:
- foreach
- Q_FOREACH
- BOOST_FOREACH
IncludeBlocks: Preserve
IncludeCategories:
- Regex: '^"(llvm|llvm-c|clang|clang-c)/'
Priority: 2
- Regex: '^(<|"(gtest|gmock|isl|json)/)'
Priority: 3
- Regex: '.*'
Priority: 1
IncludeIsMainRegex: '(Test)?$'
IndentCaseLabels: true
IndentPPDirectives: None
IndentWidth: 4
IndentWrappedFunctionNames: false
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: true
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 2
NamespaceIndentation: None
ObjCBinPackProtocolList: Auto
ObjCBlockIndentWidth: 2
ObjCSpaceAfterProperty: true
ObjCSpaceBeforeProtocolList: false
PenaltyBreakAssignment: 2
PenaltyBreakBeforeFirstCallParameter: 19
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyBreakTemplateDeclaration: 10
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerAlignment: Left
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeCpp11BracedList: false
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatements
SpaceBeforeRangeBasedForLoopColon: true
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 2
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 8
UseTab: Never
...
---
Checks: clang-diagnostic-*,
clang-analyzer-*,
clang-diagnostic-*,
clang-analyzer-*,
-clang-analyzer-security.insecureAPI.rand,
mpi-*,
readability-braces-around-statements,
readability-avoid-const-params-in-decls,
readability-redundant-string-init,
readability-container-size-empty,
readability-implicit-bool-conversion,
readability-identifier-naming,
modernize-redundant-void-arg
WarningsAsErrors: '*'
HeaderFilterRegex: ''
AnalyzeTemporaryDtors: false
FormatStyle: none
User: ExSeisDat
CheckOptions:
- key: google-readability-braces-around-statements.ShortStatementLines
value: '1'
- key: google-readability-function-size.StatementThreshold
value: '800'
- key: google-readability-namespace-comments.ShortNamespaceLines
value: '10'
- key: google-readability-namespace-comments.SpacesBeforeComments
value: '2'
- key: modernize-loop-convert.MaxCopySize
value: '16'
- key: modernize-loop-convert.MinConfidence
value: reasonable
- key: modernize-loop-convert.NamingStyle
value: CamelCase
- key: modernize-pass-by-value.IncludeStyle
value: llvm
- key: modernize-replace-auto-ptr.IncludeStyle
value: llvm
- key: modernize-use-nullptr.NullMacros
value: 'NULL'
- key: readability-braces-around-statements.ShortStatementLines
value: '0'
- key: readability-implicit-bool-conversion.AllowIntegerConditions
value: '0'
- key: readability-implicit-bool-conversion.AllowPointerConditions
value: '0'
- key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor
value: '1'
#
# Identifier Naming
#
# TODO: Add TypeDefCase, TypeAliasCase, EnumCase, EnumConstantCase, ClassCase,
# and UnionCase
- key: readability-identifier-naming.NamespaceCase
value: lower_case
- key: readability-identifier-naming.MemberCase
value: lower_case
- key: readability-identifier-naming.PrivateMemberCase
value: lower_case
- key: readability-identifier-naming.PrivateMemberPrefix
value: m_
- key: readability-identifier-naming.ProtectedMemberCase
value: lower_case
- key: readability-identifier-naming.ProtectedMemberPrefix
value: m_
- key: readability-identifier-naming.FunctionCase
value: lower_case
- key: readability-identifier-naming.ParameterCase
value: lower_case
- key: readability-identifier-naming.VariableCase
value: lower_case
- key: readability-identifier-naming.TemplateParameterCase
value: CamelCase
- key: readability-identifier-naming.MacroDefinitionCase
value: UPPER_CASE
- key: readability-identifier-naming.MacroDefinitionPrefix
value: EXSEISDAT_
...
cmake_minimum_required(VERSION 3.0)
project(fiphoboserver)
set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
find_package(proxygen REQUIRED)
add_subdirectory(src)
# FIPhoboServer
Build with
```
cmake -DDEBUG=ON -DCMAKE_PREFIX_PATH=/path/to/proxygen /path/to/CMakeLists/file/
make
```
from any directory you want and hope for the best! ;)
Oh, make sure Phobos include directories are under `../phobos/src/include` from the main `CMakeLists.txt`, or change that path in there
if(DEBUG)
add_compile_definitions(DEBUG)
endif(DEBUG)
add_subdirectory(storage)
add_subdirectory(stream)
add_subdirectory(server)
#pragma once
#include <proxygen/lib/http/HTTPMessage.h>
namespace fiphoboserver {
class S3_header {
public:
void setHeaders(std::unique_ptr<proxygen::HTTPMessage> newHeaders)
{
headers = std::move(newHeaders);
}
std::string getBucket()
{
if (!headers) {
return "";
}
std::string path = headers->getPath();
path = path.substr(1, path.size());
std::string bucketName = path.substr(0, path.find('/'));
return bucketName;
}
std::string getKey()
{
if (!headers) {
return "";
}
std::string path = headers->getPath();
path = path.substr(1, path.size());
std::string fileKey = path.substr(path.find('/') + 1, path.size() - 1);
if (fileKey.find('/') != std::string::npos
|| fileKey.find('\\') != std::string::npos) {
std::cout << "Bad Key! " << std::endl;
return "";
}
return fileKey;
}
bool isCreateBucketRequest()
{
if (!headers) {
return false;
}
if (getKey() == "") return true;
return false;
}
size_t getBodyLength()
{
std::string contentLength =
headers->getHeaders().getSingleOrEmpty("Content-Length");
return static_cast<size_t>(std::stoi(contentLength));
}
std::vector<std::pair<std::string, std::string>> getMetaData()
{
std::vector<std::pair<std::string, std::string>> metadata;
std::string metaSearchString = "x-amz-meta-";
headers->getHeaders().forEach([&](const std::string& header,
const std::string& val) {
if (header.find(metaSearchString) != std::string::npos) {
std::string metaName =
header.substr(metaSearchString.size(), header.size() - 1);
std::pair<std::string, std::string> currentPair(metaName, val);
metadata.push_back(currentPair);
}
});
return metadata;
}
private:
std::unique_ptr<proxygen::HTTPMessage> headers;
};
} // namespace fiphoboserver
include_directories(${PHOBOS_INCLUDE_DIRECTORY})
include_directories(/usr/include/glib-2.0)
include_directories(/usr/lib64/glib-2.0/include)
add_executable(
fiphoboserver
server.main.cc
get_request_handler.cc
put_request_handler.cc
unsupported_request_handler.cc
)
if(CUSTOM_OUTPUT_DIRECTORY)
set_target_properties( fiphoboserver
PROPERTIES
ARCHIVE_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver"
LIBRARY_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver"
RUNTIME_OUTPUT_DIRECTORY "${CUSTOM_OUTPUT_DIRECTORY}/fiphoboserver"
)
endif(CUSTOM_OUTPUT_DIRECTORY)
target_compile_features(fiphoboserver PUBLIC cxx_std_14)
target_link_libraries(fiphoboserver PUBLIC proxygen::proxygen)
target_link_libraries(fiphoboserver PUBLIC proxygen::proxygenhttpserver)
target_link_libraries(fiphoboserver PUBLIC stream)
target_link_libraries(fiphoboserver PUBLIC storage)
/*
* Copyright (c) Facebook, Inc. and its affiliates.
* All rights reserved.
* * This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
#include "get_request_handler.h"
#include <cstdio>
#include <stdlib.h>
#include <folly/FileUtil.h>
#include <folly/executors/GlobalExecutor.h>
#include <folly/io/async/EventBaseManager.h>
#include <proxygen/httpserver/RequestHandler.h>
#include <proxygen/httpserver/ResponseBuilder.h>
#include "../S3_header.h"
#include "../stream/FiPhoExceptions.h"
namespace fiphoboserver {
void GetRequestHandler::onRequest(
std::unique_ptr<proxygen::HTTPMessage> headers) noexcept
{
if (headers->getMethod() != proxygen::HTTPMethod::GET) {
proxygen::ResponseBuilder(downstream_)
.status(400, "Bad method")
.body("Only GET is supported")
.sendWithEOM();
return;
}
/* Creating Bucket queries */
s3_header.setHeaders(std::move(headers));
try {
/* Send meta data to backend through the stream */
stream->set_storage_meta_data(s3_header.getMetaData(), s3_header.getBucket());
/* Tell stream to coordinate with backend to prepare for GET operation */
stream->start_get(s3_header.getKey());
file_closed_ = false; // TODO: Better way of communicating this
}
catch (const std::system_error& ex) {
proxygen::ResponseBuilder(downstream_)
.status(404, "Not Found")
.body(folly::to<std::string>(
"Could not find ", headers->getPathAsStringPiece(),
" ex=", folly::exceptionStr(ex)))
.sendWithEOM();
return;
}
catch (const fiphoexceptions::PhobosException& ex) {
proxygen::ResponseBuilder(downstream_)
.status(409, "Conflict")
.body(ex.what())
.sendWithEOM();
return;
}
proxygen::ResponseBuilder(downstream_).status(200, "Ok").send();
/* Initiating a read from the stream and creating a body to send */
readFileScheduled_ = true;
folly::getCPUExecutor()->add(std::bind(
&GetRequestHandler::readFile, this,
folly::EventBaseManager::get()->getEventBase()));
}
void GetRequestHandler::readFile(folly::EventBase* evb)
{
folly::IOBufQueue buf;
while (!file_closed_ && !paused_) {
/* read 4k-ish chunks and foward each one to the client */
auto data = buf.preallocate(4000, 4000);
/*
* rc is set to error code or bytes read
* This informs the server how to proceed
* Note: zero will only be returned if the
* file is empty AND the file is closed
* for writing
*/
auto rc = stream->get(data.first, data.second);
if (rc < 0) {
// error
VLOG(4) << "Read error=" << rc;
evb->runInEventBaseThread([this] {
LOG(ERROR) << "Error reading file";
downstream_->sendAbort();
});
break;
}
else if (rc == 0) {
// done
file_closed_ = true;
VLOG(4) << "Read EOF";
evb->runInEventBaseThread([this] {
proxygen::ResponseBuilder(downstream_).sendWithEOM();
});
break;
}
else {
buf.postallocate(rc);
evb->runInEventBaseThread([this, body = buf.move()]() mutable {
proxygen::ResponseBuilder(downstream_)
.body(std::move(body))
.send();
});
}
}
/* Notify the request thread that we terminated the readFile loop */
evb->runInEventBaseThread([this] {
readFileScheduled_ = false;
if (!checkForCompletion() && !paused_) {
VLOG(4) << "Resuming deferred readFile";
onEgressResumed();
}
});
}
void GetRequestHandler::onEgressPaused() noexcept
{
/* This will terminate readFile soon */
VLOG(4) << "GetRequestHandler pause";
paused_ = true;
}
void GetRequestHandler::onEgressResumed() noexcept
{
VLOG(4) << "GetRequestHandler resumed";
paused_ = false;
/* If readFileScheduled_, it will reschedule itself */
if (!readFileScheduled_) {
readFileScheduled_ = true;
folly::getCPUExecutor()->add(std::bind(
&GetRequestHandler::readFile, this,
folly::EventBaseManager::get()->getEventBase()));
}
else {
VLOG(4) << "Deferred scheduling readFile";
}
}
void GetRequestHandler::onBody(std::unique_ptr<folly::IOBuf> /*body*/) noexcept
{
// ignore, only support GET
}
void GetRequestHandler::onEOM() noexcept
{
}
void GetRequestHandler::onUpgrade(
proxygen::UpgradeProtocol /*protocol*/) noexcept
{
// handler doesn't support upgrades
}
void GetRequestHandler::requestComplete() noexcept
{
finished_ = true;
paused_ = true;
checkForCompletion();
}
void GetRequestHandler::onError(proxygen::ProxygenError /*err*/) noexcept
{
finished_ = true;
paused_ = true;
checkForCompletion();
}
bool GetRequestHandler::checkForCompletion()
{
if (finished_ && !readFileScheduled_) {
VLOG(4) << "deleting GetRequestHandler";
delete this;
return true;
}
return false;
}
} // namespace fiphoboserver
/*
* Copyright (c) Facebook, Inc. and its affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
#pragma once
//#include <folly/File.h>
#include <folly/Memory.h>
#include <proxygen/httpserver/RequestHandler.h>
#include "../S3_header.h"
#include "../stream/stream.h"
namespace proxygen {
class ResponseHandler;
}
namespace fiphoboserver {
class GetRequestHandler : public proxygen::RequestHandler {
public:
void onRequest(
std::unique_ptr<proxygen::HTTPMessage> headers) noexcept override;
void onBody(std::unique_ptr<folly::IOBuf> body) noexcept override;
void onEOM() noexcept override;
void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override;
void requestComplete() noexcept override;
void onError(proxygen::ProxygenError err) noexcept override;
void onEgressPaused() noexcept override;
void onEgressResumed() noexcept override;
GetRequestHandler(std::unique_ptr<Stream> input_stream):
stream(std::move(input_stream)) {}
private:
void readFile(folly::EventBase* evb);
bool checkForCompletion();
std::unique_ptr<Stream> stream;
S3_header s3_header;
bool readFileScheduled_{false};
std::atomic<bool> paused_{false};
bool finished_{false};
bool file_closed_{false};
};
} // namespace fiphoboserver
/*
* Copyright (c) Facebook, Inc. and its affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
#include "put_request_handler.h"
#include <folly/FileUtil.h>
#include <proxygen/httpserver/PushHandler.h>
#include <proxygen/httpserver/RequestHandler.h>
#include <proxygen/httpserver/ResponseBuilder.h>
#include "../S3_header.h"
#include "../stream/FiPhoExceptions.h"
namespace fiphoboserver {