Commit fa808a8d authored by Sophie Wenzel-Teuber's avatar Sophie Wenzel-Teuber
Browse files

Update to Phobos version 1.92

Changelog:
* Update wrapper to new function signatures and structs
* Run format and lint
* Temporarily remove Centos7 build from CI
parent 735d8bae
Pipeline #3707 passed with stages
in 14 minutes and 36 seconds
image: ciaranorourke/deimos:centos8 image: swenzelteuber/superdeimos:centos8
.snippets: .snippets:
- &start_phobos - &start_phobos
...@@ -58,6 +58,7 @@ image: ciaranorourke/deimos:centos8 ...@@ -58,6 +58,7 @@ image: ciaranorourke/deimos:centos8
- dnf install -y - dnf install -y
cmake cmake
clang clang-tools-extra clang clang-tools-extra
libasan libubsan
- pip3 install boto3 - pip3 install boto3
- source ${ENV_LIST} - source ${ENV_LIST}
script: script:
...@@ -141,20 +142,20 @@ Authorisation Disabled: ...@@ -141,20 +142,20 @@ Authorisation Disabled:
<<: *default_cmake_variables <<: *default_cmake_variables
DEIMOS_AUTHORISATION: "OFF" DEIMOS_AUTHORISATION: "OFF"
Centos7: # Centos7:
<<: *default_job # <<: *default_job
stage: test # stage: test
image: ciaranorourke/deimos:centos7 # image: ciaranorourke/deimos:centos7
variables: # variables:
<<: *default_cmake_variables # <<: *default_cmake_variables
ENV_LIST: /home/superdeimos/centos7_envs.list # ENV_LIST: /home/superdeimos/centos7_envs.list
CMAKE_BUILD_TYPE: Release # CMAKE_BUILD_TYPE: Release
before_script: # before_script:
- yum update -y # - yum update -y
- yum install -y epel-release # - yum install -y epel-release
- yum install -y # - yum install -y
cmake3 # cmake3
python3 # python3
- pip3 install boto3 # - pip3 install boto3
- ln -s /usr/bin/cmake3 /usr/bin/cmake # - ln -s /usr/bin/cmake3 /usr/bin/cmake
- source ${ENV_LIST} # - source ${ENV_LIST}
...@@ -569,7 +569,7 @@ the signature stored in the request ...@@ -569,7 +569,7 @@ the signature stored in the request
[Go to Top](#deimos-protocol-s3-S3_authorisation) [Go to Top](#deimos-protocol-s3-S3_authorisation)
### <a name='deimos-protocol-s3-S3_authorisation-m_payload' /> private deimos::protocol::s3::S3_authorisation::m_payload = "" ### <a name='deimos-protocol-s3-S3_authorisation-m_payload' /> private deimos::protocol::s3::S3_authorisation::m_payload
the payload of the request the payload of the request
......
...@@ -24,7 +24,7 @@ class DeimosException : public std::exception { ...@@ -24,7 +24,7 @@ class DeimosException : public std::exception {
/// ///
/// @brief default constructor /// @brief default constructor
/// ///
DeimosException() : m_message("") {} DeimosException() {}
/// ///
/// @brief constructor for a message with information on the exception /// @brief constructor for a message with information on the exception
......
...@@ -215,7 +215,7 @@ class S3_authorisation : public Authorisation { ...@@ -215,7 +215,7 @@ class S3_authorisation : public Authorisation {
/// ///
/// This is the whole body since we do not support multiple chunked /// This is the whole body since we do not support multiple chunked
/// signatures /// signatures
std::string m_payload = ""; std::string m_payload;
/// @brief the error description if one occurred /// @brief the error description if one occurred
S3_error_info m_error = access_denied; S3_error_info m_error = access_denied;
}; };
......
...@@ -74,24 +74,24 @@ const S3_error_info incomplete_body = { ...@@ -74,24 +74,24 @@ const S3_error_info incomplete_body = {
const S3_error_info internal_error = { const S3_error_info internal_error = {
500, "InternalError", 500, "InternalError",
"We encountered an internal error. Please try again."}; "We encountered an internal error. Please try again."};
const S3_error_info invalid_argument = {400, "InvalidArgument", const S3_error_info invalid_argument = {
"Invalid Argument"}; 400, "InvalidArgument", "Invalid Argument"};
const S3_error_info invalid_bucketname = {400, "InvalidBucketName", const S3_error_info invalid_bucketname = {
"The specified bucket is not valid."}; 400, "InvalidBucketName", "The specified bucket is not valid."};
const S3_error_info invalid_key = {403, "InvalidAccessKeyId", const S3_error_info invalid_key = {
"The given key is not valid"}; 403, "InvalidAccessKeyId", "The given key is not valid"};
const S3_error_info invalid_signature_type = {400, "InvalidRequest", const S3_error_info invalid_signature_type = {
"Please use AWS4-HMAC-SHA256."}; 400, "InvalidRequest", "Please use AWS4-HMAC-SHA256."};
const S3_error_info malformed_authorisation = { const S3_error_info malformed_authorisation = {
400, "AuthorizationHeaderMalformed", 400, "AuthorizationHeaderMalformed",
"The authorization header you provided is invalid."}; "The authorization header you provided is invalid."};
const S3_error_info missing_content_length = { const S3_error_info missing_content_length = {
411, "MissingContentLength", 411, "MissingContentLength",
"You must provide the Content-Length HTTP header."}; "You must provide the Content-Length HTTP header."};
const S3_error_info no_such_bucket = {404, "NoSuchBucket", const S3_error_info no_such_bucket = {
"The specified bucket does not exist."}; 404, "NoSuchBucket", "The specified bucket does not exist."};
const S3_error_info no_such_key = {404, "NoSuchKey", const S3_error_info no_such_key = {
"The specified key does not exist."}; 404, "NoSuchKey", "The specified key does not exist."};
const S3_error_info no_such_version = { const S3_error_info no_such_version = {
404, "NoSuchVersion", 404, "NoSuchVersion",
"The version ID specified in the request does not match any existing version. "}; "The version ID specified in the request does not match any existing version. "};
......
...@@ -44,14 +44,17 @@ void pho_xfer_desc_destroy_cpp(struct pho_xfer_desc* xfer) ...@@ -44,14 +44,17 @@ void pho_xfer_desc_destroy_cpp(struct pho_xfer_desc* xfer)
} }
int phobos_store_object_list_cpp( int phobos_store_object_list_cpp(
const char* pattern, const char** res,
int n_res,
bool is_pattern,
const char** metadata, const char** metadata,
int n_metadata, int n_metadata,
bool deprecated,
struct object_info** objs, struct object_info** objs,
int* n_objs) int* n_objs)
{ {
return phobos_store_object_list( return phobos_store_object_list(
pattern, metadata, n_metadata, objs, n_objs); res, n_res, is_pattern, metadata, n_metadata, deprecated, objs, n_objs);
} }
void phobos_store_object_list_free_cpp(struct object_info* objs, int n_objs) void phobos_store_object_list_free_cpp(struct object_info* objs, int n_objs)
......
...@@ -28,11 +28,7 @@ enum rsc_family { ...@@ -28,11 +28,7 @@ enum rsc_family {
PHO_RSC_UNSPEC = PHO_RSC_LAST, PHO_RSC_UNSPEC = PHO_RSC_LAST,
}; };
static const char* const rsc_family_names[] = { static const char* const rsc_family_names[] = {"disk", "tape", "dir"};
"disk",
"tape",
"dir",
};
static inline const char* rsc_family2str(enum rsc_family family) static inline const char* rsc_family2str(enum rsc_family family)
{ {
...@@ -47,9 +43,7 @@ static inline enum rsc_family str2rsc_family(const char* str) ...@@ -47,9 +43,7 @@ static inline enum rsc_family str2rsc_family(const char* str)
int i; int i;
for (i = 0; i < PHO_RSC_LAST; i++) { for (i = 0; i < PHO_RSC_LAST; i++) {
if (strcmp(str, rsc_family_names[i]) == 0) { if (!strcmp(str, rsc_family_names[i])) return rsc_family(i); // NOLINT
return (rsc_family)i;
}
} }
return PHO_RSC_INVAL; return PHO_RSC_INVAL;
} }
...@@ -67,9 +61,13 @@ struct tags { ...@@ -67,9 +61,13 @@ struct tags {
* Exact semantic depends on the operation it is applied on. * Exact semantic depends on the operation it is applied on.
*/ */
enum pho_xfer_flags { enum pho_xfer_flags {
/* put: replace the object if it already exists (_not supported_) /**
* get: replace the target file if it already exists */ * put: replace the object if it already exists (_not supported_)
* get: replace the target file if it already exists
*/
PHO_XFER_OBJ_REPLACE = (1 << 0), PHO_XFER_OBJ_REPLACE = (1 << 0),
/* get: check the object's location before getting it */
PHO_XFER_OBJ_BEST_HOST = (1 << 1),
}; };
/** /**
...@@ -87,17 +85,39 @@ typedef void (*pho_completion_cb_t)(void* u, const struct pho_xfer_desc*, int); ...@@ -87,17 +85,39 @@ typedef void (*pho_completion_cb_t)(void* u, const struct pho_xfer_desc*, int);
enum pho_xfer_op { enum pho_xfer_op {
PHO_XFER_OP_PUT, /**< PUT operation. */ PHO_XFER_OP_PUT, /**< PUT operation. */
PHO_XFER_OP_GET, /**< GET operation. */ PHO_XFER_OP_GET, /**< GET operation. */
PHO_XFER_OP_GETMD /**< GET metadata operation. */ PHO_XFER_OP_GETMD, /**< GET metadata operation. */
PHO_XFER_OP_DEL, /**< DEL operation. */
PHO_XFER_OP_UNDEL, /**< UNDEL operation. */
PHO_XFER_OP_LAST
}; };
/** /**
* PUT parameters. * PUT parameters.
* Family, layout_name and tags can be set directly or by using an alias.
* An alias is a name defined in the phobos config to combine these parameters.
* The alias will not override family and layout if they have been specified
* in this struct but extend existing tags.
*/ */
struct pho_xfer_put_params { struct pho_xfer_put_params {
ssize_t size; /**< Amount of data to write. */ ssize_t size; /**< Amount of data to write. */
enum rsc_family family; /**< Targeted resource family. */ enum rsc_family family; /**< Targeted resource family. */
const char* layout_name; /**< Name of the layout module to use. */ const char* layout_name; /**< Name of the layout module to use. */
struct tags tags; /**< Tags to select a media to write. */ struct tags tags; /**< Tags to select a media to write. */
const char* alias; /**< Identifier for family, layout,
* tag combination
*/
bool overwrite; /**< true if the put command could be an
* update.
*/
};
/**
* GET parameters.
* Node_name corresponds to the name of the node the object can be retrieved
* from, if a phobos_get call fails.
*/
struct pho_xfer_get_params {
char* node_name; /**< Node name [out] */
}; };
/** /**
...@@ -105,8 +125,10 @@ struct pho_xfer_put_params { ...@@ -105,8 +125,10 @@ struct pho_xfer_put_params {
*/ */
union pho_xfer_params { union pho_xfer_params {
struct pho_xfer_put_params put; /**< PUT parameters. */ struct pho_xfer_put_params put; /**< PUT parameters. */
struct pho_xfer_get_params get; /**< GET parameters */
}; };
/** /**
* Xfer descriptor. * Xfer descriptor.
* The source/destination semantics of the fields vary * The source/destination semantics of the fields vary
...@@ -115,9 +137,12 @@ union pho_xfer_params { ...@@ -115,9 +137,12 @@ union pho_xfer_params {
* - pĥobos_getmd() * - pĥobos_getmd()
* - phobos_get() * - phobos_get()
* - phobos_put() * - phobos_put()
* - phobos_undelete()
*/ */
struct pho_xfer_desc { struct pho_xfer_desc {
char* xd_objid; /**< Object ID to read or write. */ char* xd_objid; /**< Object ID to read or write. */
char* xd_objuuid; /**< Object UUID to read or write. */
int xd_version; /**< Object version. */
enum pho_xfer_op xd_op; /**< Operation to perform. */ enum pho_xfer_op xd_op; /**< Operation to perform. */
int xd_fd; /**< FD of the source/destination. */ int xd_fd; /**< FD of the source/destination. */
struct pho_attrs xd_attrs; /**< User defined attributes. */ struct pho_attrs xd_attrs; /**< User defined attributes. */
...@@ -128,7 +153,10 @@ struct pho_xfer_desc { ...@@ -128,7 +153,10 @@ struct pho_xfer_desc {
struct object_info { struct object_info {
char* oid; char* oid;
char* uuid;
int version;
char* user_md; char* user_md;
struct timeval deprec_time;
}; };
/** /**
...@@ -209,12 +237,21 @@ void pho_xfer_desc_destroy_cpp(struct pho_xfer_desc* xfer); ...@@ -209,12 +237,21 @@ void pho_xfer_desc_destroy_cpp(struct pho_xfer_desc* xfer);
/** /**
* Retrieve the objects that match the given pattern and metadata. * Retrieve the objects that match the given pattern and metadata.
* If given multiple objids or patterns, retrieve every item with name
* matching any of those objids or patterns.
* If given multiple objids or patterns, and metadata, retrieve every item
* with name matching any of those objids or pattersn, but containing
* every given metadata.
* *
* The caller must release the list calling phobos_store_object_list_free(). * The caller must release the list calling phobos_store_object_list_free().
* *
* \param[in] pattern Shell regexp pattern. * \param[in] res Objids or patterns, depending on
* \a is_pattern.
* \param[in] n_res Number of requested objids or patterns.
* \param[in] is_pattern True if search using POSIX pattern.
* \param[in] metadata Metadata filter. * \param[in] metadata Metadata filter.
* \param[in] n_metadata Number of requested metadata. * \param[in] n_metadata Number of requested metadata.
* \param[in] deprecated true if search from deprecated objects.
* \param[out] objs Retrieved objects. * \param[out] objs Retrieved objects.
* \param[out] n_objs Number of retrieved items. * \param[out] n_objs Number of retrieved items.
* *
...@@ -222,9 +259,12 @@ void pho_xfer_desc_destroy_cpp(struct pho_xfer_desc* xfer); ...@@ -222,9 +259,12 @@ void pho_xfer_desc_destroy_cpp(struct pho_xfer_desc* xfer);
* -errno on failure. * -errno on failure.
*/ */
int phobos_store_object_list_cpp( int phobos_store_object_list_cpp(
const char* pattern, const char** res,
int n_res,
bool is_pattern,
const char** metadata, const char** metadata,
int n_metadata, int n_metadata,
bool deprecated,
struct object_info** objs, struct object_info** objs,
int* n_objs); int* n_objs);
......
...@@ -203,7 +203,7 @@ std::vector<std::map<std::string, std::string>> Phobos_file::get_object_list( ...@@ -203,7 +203,7 @@ std::vector<std::map<std::string, std::string>> Phobos_file::get_object_list(
const char* pattern = ""; const char* pattern = "";
int rc = phobos_store_object_list_cpp( int rc = phobos_store_object_list_cpp(
pattern, &meta_data, n_meta_data, &obj_info, &n_objs); &pattern, 1, true, &meta_data, n_meta_data, false, &obj_info, &n_objs);
if (rc != 0) { if (rc != 0) {
throw PhobosException( throw PhobosException(
......
...@@ -41,8 +41,8 @@ class Fifo : public Stream { ...@@ -41,8 +41,8 @@ class Fifo : public Stream {
/// ///
Fifo(Fifo&& input) : Fifo(Fifo&& input) :
Stream(std::move(input.m_storage)), Stream(std::move(input.m_storage)),
m_fifo_descriptor{input.m_fifo_descriptor[0], m_fifo_descriptor{
input.m_fifo_descriptor[1]} input.m_fifo_descriptor[0], input.m_fifo_descriptor[1]}
{ {
} }
......
...@@ -70,8 +70,8 @@ cli_options Config::handle_arguments() ...@@ -70,8 +70,8 @@ cli_options Config::handle_arguments()
file.close(); file.close();
} }
return {m_http_port, m_http2_port, m_hostname, m_thread_count, return {
m_log_filename}; m_http_port, m_http2_port, m_hostname, m_thread_count, m_log_filename};
} }
} // namespace util } // namespace util
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment