Commit 2aa29e15 authored by Pádraig Ó Conbhuí's avatar Pádraig Ó Conbhuí
Browse files

Add Deimos 5f97c4546617aca666bd6f217f81af78607830c4 to external/deimos

parents 8af514ee 6499dfb0
---
Language: Cpp
AccessModifierOffset: -2
AlignAfterOpenBracket: AlwaysBreak
AlignConsecutiveAssignments: true
AlignConsecutiveDeclarations: false
AlignEscapedNewlines: Right
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: Yes
BinPackArguments: true
BinPackParameters: false
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
AfterExternBlock: false
BeforeCatch: true
BeforeElse: true
IndentBraces: false
SplitEmptyFunction: true
SplitEmptyRecord: true
SplitEmptyNamespace: true
BreakBeforeBinaryOperators: NonAssignment
BreakBeforeBraces: Stroustrup
BreakBeforeInheritanceComma: false
BreakInheritanceList: AfterColon
BreakBeforeTernaryOperators: false
BreakConstructorInitializersBeforeComma: false
BreakConstructorInitializers: AfterColon
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: false
ColumnLimit: 80
CommentPragmas: '^ *@'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
ForEachMacros:
- foreach
- Q_FOREACH
- BOOST_FOREACH
IncludeBlocks: Preserve
IncludeCategories:
- Regex: '^"(llvm|llvm-c|clang|clang-c)/'
Priority: 2
- Regex: '^(<|"(gtest|gmock|isl|json)/)'
Priority: 3
- Regex: '.*'
Priority: 1
IncludeIsMainRegex: '(Test)?$'
IndentCaseLabels: true
IndentPPDirectives: None
IndentWidth: 4
IndentWrappedFunctionNames: false
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: true
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 2
NamespaceIndentation: None
ObjCBinPackProtocolList: Auto
ObjCBlockIndentWidth: 2
ObjCSpaceAfterProperty: true
ObjCSpaceBeforeProtocolList: false
PenaltyBreakAssignment: 2
PenaltyBreakBeforeFirstCallParameter: 19
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyBreakTemplateDeclaration: 10
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerAlignment: Left
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeCpp11BracedList: false
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatements
SpaceBeforeRangeBasedForLoopColon: true
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 2
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 8
UseTab: Never
...
---
Checks: clang-diagnostic-*,
clang-analyzer-*,
clang-diagnostic-*,
clang-analyzer-*,
-clang-analyzer-security.insecureAPI.rand,
mpi-*,
readability-braces-around-statements,
readability-redundant-string-init,
readability-container-size-empty,
readability-implicit-bool-conversion,
readability-identifier-naming,
modernize-redundant-void-arg
WarningsAsErrors: '*'
HeaderFilterRegex: ''
AnalyzeTemporaryDtors: false
FormatStyle: none
User: Deimos
CheckOptions:
- key: google-readability-braces-around-statements.ShortStatementLines
value: '1'
- key: google-readability-function-size.StatementThreshold
value: '800'
- key: google-readability-namespace-comments.ShortNamespaceLines
value: '10'
- key: google-readability-namespace-comments.SpacesBeforeComments
value: '2'
- key: modernize-loop-convert.MaxCopySize
value: '16'
- key: modernize-loop-convert.MinConfidence
value: reasonable
- key: modernize-loop-convert.NamingStyle
value: CamelCase
- key: modernize-pass-by-value.IncludeStyle
value: llvm
- key: modernize-replace-auto-ptr.IncludeStyle
value: llvm
- key: modernize-use-nullptr.NullMacros
value: 'NULL'
- key: readability-braces-around-statements.ShortStatementLines
value: '0'
- key: readability-implicit-bool-conversion.AllowIntegerConditions
value: '0'
- key: readability-implicit-bool-conversion.AllowPointerConditions
value: '0'
- key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor
value: '1'
#
# Identifier Naming
#
# TODO: Add TypeDefCase, TypeAliasCase, EnumCase, EnumConstantCase, ClassCase,
# and UnionCase
- key: readability-identifier-naming.NamespaceCase
value: lower_case
- key: readability-identifier-naming.MemberCase
value: lower_case
- key: readability-identifier-naming.PrivateMemberCase
value: lower_case
- key: readability-identifier-naming.PrivateMemberPrefix
value: m_
- key: readability-identifier-naming.ProtectedMemberCase
value: lower_case
- key: readability-identifier-naming.ProtectedMemberPrefix
value: m_
- key: readability-identifier-naming.FunctionCase
value: lower_case
- key: readability-identifier-naming.ParameterCase
value: lower_case
- key: readability-identifier-naming.VariableCase
value: lower_case
- key: readability-identifier-naming.TemplateParameterCase
value: CamelCase
- key: readability-identifier-naming.MacroDefinitionCase
value: UPPER_CASE
- key: readability-identifier-naming.MacroDefinitionPrefix
value: DEIMOS_
...
*.swp
todo.txt
.DS_Store
image: ciaranorourke/fiphoboserver:debian
.snippets:
- &start_phobos
/home/superfiphoboserver/tools/build_phobos.sh -r
- &configure
mkdir build
&& pushd build
&& (
set -x;
cmake
-DCMAKE_VERBOSE_MAKEFILE=ON
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX}
-DDEIMOS_BUILD_TESTS="${DEIMOS_BUILD_TESTS}"
-DCMAKE_PREFIX_PATH="${DEPS_DIR}"
-DCUSTOM_DOCU_PATH="${DEIMOS_DOC_DIR}"
-DDEIMOS_BUILD_DOCUMENTATION="${DEIMOS_BUILD_DOCUMENTATION}"
-DDEIMOS_DISABLE_AUTHORISATION="${DEIMOS_DISABLE_AUTHORISATION}"
..
)
&& popd
- &build
make -C build
- &unit_test
make -C build test ARGS=-V
- &integration_test
cd test/integration_tests
&& python3 -m unittest client
.cmake_variables:
default_cmake_variables: &default_cmake_variables
CC: gcc
CXX: g++
DEPS_DIR: /home/superfiphoboserver/build/install
DEIMOS_DOC_DIR: ""
DEIMOS_BUILD_TESTS: "ON"
DEIMOS_DISABLE_AUTHORISATION: "OFF"
.default_job: &default_job
tags:
- docker
variables:
<<: *default_cmake_variables
before_script:
- apt-get update
- apt-get install -y
cmake
clang clang-tidy
libspdlog-dev
script:
- *start_phobos
- *configure
- *build
- *unit_test
- *integration_test
stages:
- static analysis
- build
- test
Format:
<<: *default_job
stage: static analysis
variables:
CC: clang
CXX: clang++
before_script:
- apt-get update
- apt-get install -y
git
clang-format
- pip3 install yapf
script:
- ./tools/run_format.sh
&& git status
&& git diff-index --quiet HEAD
|| { echo "This commit contains unformatted files! Run tools/run_format.sh on the project to format them correctly."; false; }
Lint:
<<: *default_job
stage: static analysis
variables:
<<: *default_cmake_variables
CC: clang
CXX: clang++
script:
- *configure
- *build
- ./tools/run_lint.sh build
Documentation:
<<: *default_job
stage: build
variables:
<<: *default_cmake_variables
DEIMOS_BUILD_DOCUMENTATION: "ON"
DEIMOS_DOC_DIR: /home/superfiphoboserver/external/doxygen-xml-parser/src
CC: clang
CXX: clang++
before_script:
- apt-get update
- apt-get install -y
cmake
doxygen
graphviz
clang
script:
- *configure
- make -C build/doc
&& git status
&& git diff-index --quiet HEAD
|| { echo "This commit contains changes to the documentation! Run doxygen-XML-parser on the project update the documentation."; false; }
Default:
<<: *default_job
stage: test
Authorisation Disabled:
<<: *default_job
variables:
<<: *default_cmake_variables
DEIMOS_DISABLE_AUTHORISATION: "ON"
stage: test
cmake_minimum_required(VERSION 3.0)
project(deimos)
set(CUSTOM_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
find_package(proxygen REQUIRED)
#
# Provide options to build or skip building the
# programs/libraries in the given subdirectories
#
option(
DEIMOS_BUILD_TESTS
"build Deimos tests. Depends on Catch2."
OFF
)
option(
DEIMOS_BUILD_DOCUMENTATION
"build Deimos Documentation. Depends on Doxygen."
OFF
)
option(
DEIMOS_DISABLE_AUTHORISATION
"build Deimos with AWS authorisation disabled. Default is enabled"
OFF
)
add_subdirectory(src)
target_include_directories(
server SYSTEM PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>
)
if(DEIMOS_BUILD_DOCUMENTATION)
add_subdirectory(doc)
endif(DEIMOS_BUILD_DOCUMENTATION)
# Build tests
if(DEIMOS_BUILD_TESTS)
add_subdirectory(test/unit_tests)
enable_testing()
endif(DEIMOS_BUILD_TESTS)
# Deimos
## Dependencies Installation Instructions
Deimos has many dependencies that require much patience to build. It may be simpler to invoke a prebuilt Docker image using a Dockerfile such as;
```
FROM ciaranorourke/deimos:debian
RUN apt-get -y update && apt-get -y upgrade
RUN apt-get -y install doxygen graphviz
RUN apt-get -y install \
[personal installation preferences]
CMD /home/superdeimos/tools/build_phobos.sh -r && [favourite shell]
```
Alternatively, see [SuperDeimos](https://git.ichec.ie/oilgas/ltfs/superdeimos) for instructions on how to create your own build environment.
It is recommended to run this Docker image by mounting your local branch of the Deimos project with `docker run -it -v /path/to/deimos:/destination [name]:[tag]`
To use the Phobos raid1 layout which is compatible with FIFOs, the following environment variables must be set;
```
export PHOBOS_STORE_layout=raid1
export PHOBOS_LAYOUT_RAID1_repl_count={1,2,3,...}
```
## Build Instructions
To configure and build Deimos, a C++14 compatible compiler is required, and CMake >= 3.0.
```
# Make a directory to hold the temporary build files
mkdir -p build
# Change directory to the build directory
cd build
# Configure the Deimos build using CMake
cmake \
-DCMAKE_PREFIX_PATH=/path/to/proxygen \
[more options below] \
/path/to/deimos/project
# Build the Deimos project
make
```
### Build instructions using the superbuild
When using the Docker image from [SuperDeimos](https://git.ichec.ie/oilgas/ltfs/superdeimos), some specific values make life a bit easier:
* docker image should be run with `/home/superdeimos/external/deimos` as `destination` for the mount option (example: `docker run -it -v ./deimos:/home/superdeimos/external/deimos phobos:debian`).
* the `CMAKE_PREFIX_PATH` need to be set to `/home/superdeimos/build/install`
* An example script to build this project can be found inside the container under `/home/superdeimos/tools/build_deimos.sh`.
### Useful Options
#### Setting Compile/Link Flags
Option | Effect
------ | ------
`-DCMAKE_CXX_COMPILER=...` | Set the C++ compiler.
`-DCMAKE_CXX_FLAGS=...` | Set the flags to pass to the C++ compiler. Overrides the default flags.
#### Enabling/Disabling Sections of Deimos
Option | Effect
------ | ------
`-DDEIMOS_BUILD_TESTS=...` | Set to `ON` to build Deimos tests and enable the `make test` target, or `OFF` to skip (Default `OFF`).
`-DDEIMOS_DISABLE_AUTHORISATION=...` | Set to `ON` to disable AWS V4 authorisation (= letting everything through), or `OFF` to authorise requests (Default `OFF`).
`-DDEIMOS_BUILD_DOCUMENTATION=...` | Set to `ON` to build Deimos Doxygen documentation, or `OFF` to skip (Default `OFF`).
`-DCUSTOM_DOCU_PATH=...` | Set to the path to the `src` folder in [DoxygenXMLParser](https://git.ichec.ie/sophie.wenzel-teuber/doxygen-xml-parser) to create the Markdown output from the Doxygen XML, that can also be found [here](doc/Markdown/deimos) or don't set to skip (Default `not set`).
## Testing
Units test can be run with `make -C build tests`, given `DEIMOS_BUILD_TESTS` was set to `ON` during the build process.
Integration tests can be run from the `test/integration_tests/` directory with `python -m unittest client`.
## Documentation
The Doxygen documentation can be found [here](doc/Markdown/deimos).
## Usage
The `deimos` executable requires exactly one subcommand.
#### Subcommands
Subcommand | Effect
---------- | ------
`start [hostname]` | Start the server daemon, listening to `hostname`.
`stop` | Stop the server daemon.
The `start` subcommand supports a number of optional flags and options.
#### Flags/Options
Flag/Option | Effect
----------- | ------
`--http INT` | Set the port to listen on with HTTP protocol (Default `11000`).
`--http2 INT` | Set the port to listen on with HTTP2 protocol (Default `11002`).
`-t,--threads INT` | Set the number of threads to listen on, <= 0 will use number of available cores (Default `0`).
`-w,--write-config TEXT` | Write the configuration to `TEXT`.
`-l,--logfile` | Set the filename to write logs to (Default `/tmp/deimos_log.txt").
Options can also be given to the `start` subcommand using `--config [config_filename]`. Note that if `hostname` is specified in the configuration file then the `--config` options must be suppied before the `start` subcommand;
```
deimos --config [config_filename] start
```
### Authentication
The server uses the [Amazon S3 Authentication](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) to ensure the requests that are reaching it are from legitimate users.
Since we do not have a proper database for users yet, they are currently stored along with their keys in [users.txt](test/users.txt). This file is copied into the `/tmp` folder by CMake and then read from there for authentication.
Since the authentication is checked for every request, you need to use one of the users and keys from that file for testing the server!
One possible way most SDKs use automatically is the file `~/.aws/credentials`. The tests for the server are done with a file like this:
```
[default]
aws_access_key_id = OPEN_KEY
aws_secret_access_key = SECRET_KEY
```
find_package(Doxygen REQUIRED)
# Write a Doxyfile including the main doc/Doxyfile.
# This sets the @INCLUDE_PATH variable to the project root.
configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in
${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
@ONLY
)
set(INCLUDE_DOXYFILE "@INCLUDE = ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile")
# Custom command to run doxygen on build/doc/Doxyfile in the build/doc dir.
# Running doxygen is tied to the html/index.html file.
# TODO: The main dependency is currently doc/Doxyfile, but there should be
# a depencency on *every* file being documented!
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/xml/index.xml
COMMAND ${DOXYGEN_EXECUTABLE} "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile"
MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Building the documentation..."
)
if(CUSTOM_DOCU_PATH)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/docu.json
COMMAND python3
ARGS "${CUSTOM_DOCU_PATH}/xml_to_json_parser/xml_to_json_parser.py" "${CMAKE_CURRENT_BINARY_DIR}/xml" docu.json
MAIN_DEPENDENCY ${CMAKE_CURRENT_BINARY_DIR}/xml/index.xml
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Parsing the documentation XML..."
)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/Markdown/index.md
COMMAND python3
ARGS "${CUSTOM_DOCU_PATH}/json_to_Markdown_converter/json_to_Markdown_converter.py" docu.json "${CMAKE_CURRENT_SOURCE_DIR}/Markdown"
MAIN_DEPENDENCY ${CMAKE_CURRENT_BINARY_DIR}/docu.json
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Creating the Markdown documentation..."
)
# Target to wire up building the documentation to the ALL target.
add_custom_target(
doc ALL
DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/Markdown/index.md
)
else(CUSTOM_DOCU_PATH)
# Target to wire up building the documentation to the ALL target.
add_custom_target(
doc ALL
DEPENDS
${CMAKE_CURRENT_BINARY_DIR}/xml/index.xml
)
endif(CUSTOM_DOCU_PATH)
# Doxyfile 1.8.13
#
# This is a template file used for configuration in doc/CMakeLists.txt.
#
PROJECT_NAME = Deimos
# Extraction options
EXTRACT_PRIVATE = YES
EXTRACT_PACKAGE = YES
EXTRACT_STATIC = YES
EXTRACT_LOCAL_METHODS = YES
EXTRACT_ANON_NSPACES = YES
EXCLUDE_SYMBOLS = detail
# Generate case sensitive filenames
CASE_SENSE_NAMES = YES
# Warning options
# Warn on everything possible, and fail on warning
QUIET = YES
WARNINGS = YES
WARN_IF_UNDOCUMENTED = NO
WARN_IF_DOC_ERROR = YES
WARN_NO_PARAMDOC = YES
WARN_AS_ERROR = YES
# Input directories
INPUT = @CMAKE_SOURCE_DIR@/src
EXCLUDE = @CMAKE_SOURCE_DIR@/src/main.cc
EXCLUDE_PATTERNS = */phobos_cpp_wrapper/*
RECURSIVE = YES
# HTML output options
GENERATE_HTML = YES
HTML_OUTPUT = html
HTML_FILE_EXTENSION = .html
GENERATE_TREEVIEW = YES
# Latex font size
FORMULA_FONTSIZE = 10
# Macro options
ENABLE_PREPROCESSING = YES
MACRO_EXPANSION = YES
PREDEFINED = __cplusplus
SEARCH_INCLUDES = YES
GENERATE_XML = YES
# <a name='deimos-DeimosException' /> public deimos::DeimosException
exception class for all user defined exceptions in Deimos
## Inheritance:
Is inherited by [deimos::IOException][deimos-IOException], [deimos::storage::PhobosException][deimos-storage-PhobosException].
## Protected Attributes
| Name | Description |
| ---- | ---- |
| [m_message](#deimos-DeimosException-m_message) | the internal message |
## Public Functions
| Name | Description |
| ---- | ---- |
| [DeimosException](#deimos-DeimosException-DeimosException) | default constructor |
| [DeimosException](#deimos-DeimosException-DeimosException-1) | constructor for a message with information on the exception |
| [what](#deimos-DeimosException-what) | get information on this exception |
| [get_inner_error](#deimos-DeimosException-get_inner_error) | return the inner error value that caused this exception |
## Protected Attributes
### <a name='deimos-DeimosException-m_message' /> protected deimos::DeimosException::m_message
the internal message