Commit 4447cb00 authored by Aaron Dees's avatar Aaron Dees
Browse files

Merge branch '33-add-integration-tests' into 'devel'

Resolve "Add Integration Tests"

See merge request oilgas/ltfs/fiphoboserver!25
parents 88a6e21c 9569a15f
Pipeline #1238 passed with stages
in 5 minutes and 47 seconds
......@@ -21,8 +21,11 @@ image: ciaranorourke/fiphoboserver:debian
&& popd
- &build
make -C build
- &test
- &unit_test
make -C build test ARGS=-V
- &integration_test
cd test/integration_tests
&& python3 -m unittest client
.cmake_variables:
default_cmake_variables: &default_cmake_variables
......@@ -42,7 +45,8 @@ image: ciaranorourke/fiphoboserver:debian
script:
- *configure
- *build
- *test
- *unit_test
- *integration_test
stages:
- static analysis
......@@ -71,12 +75,12 @@ Clang Tidy:
stage: static analysis
variables:
<<: *default_cmake_variables
FIPHOBOSERVER_BUILD_TESTS: "ON"
CC: clang
CXX: clang++
script:
- *configure
- *build
- make -C build/test
- ./tools/run_lint.sh build
Doxygen:
......@@ -110,5 +114,17 @@ Unit Tests:
- *start_phobos
- *configure
- *build
- *test
- *unit_test
Integration Tests:
<<: *default_job
stage: test
variables:
<<: *default_cmake_variables
CC: gcc
CXX: g++
script:
- *start_phobos
- *configure
- *build
- *integration_test
......@@ -24,6 +24,6 @@ endif(CUSTOM_DOCU_PATH)
# Build tests
if(FIPHOBOSERVER_BUILD_TESTS)
add_subdirectory(test)
add_subdirectory(test/unit_tests)
enable_testing()
endif(FIPHOBOSERVER_BUILD_TESTS)
import unittest
import subprocess
import signal
import os
import utils.s3_client as s3_client
import utils.compare as compare
import utils.random_string as random_string
class Server_test_case(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.bucket_name = "ichec.phobos.7a3d12c6-632b-4291-b87e-07badca60cd0"
cls.client = s3_client.S3_client('http://localhost:11000/')
cls.server = subprocess.Popen('../../build/fiphoboserver', shell=True, preexec_fn=os.setsid)
cls.put_file_name = 'data/EmperorWu.txt'
cls.get_file_name = 'data/EmperorWu_get.txt'
cls.put_empty_file_name = 'data/empty.txt'
cls.get_empty_file_name = 'data/empty_get.txt'
cls.key = random_string.random_string('key_', 12)
@classmethod
def tearDownClass(cls):
os.remove(cls.get_file_name)
cls.server.terminate()
# kill the fiphoboserver process
os.killpg(os.getpgid(cls.server.pid), signal.SIGTERM)
def test_file_put_and_get(self):
self.client.put(
self.put_file_name,
self.bucket_name,
self.key
)
self.client.get(
self.get_file_name,
self.bucket_name,
self.key
)
self.assertTrue(
compare.files_are_equal(self.put_file_name, self.get_file_name)
)
def test_empty_file_put_and_get(self):
key = random_string.random_string('key_', 12)
self.client.put(
self.put_empty_file_name,
self.bucket_name,
key
)
self.client.get(
self.get_empty_file_name,
self.bucket_name,
key
)
self.assertTrue(
compare.files_are_equal(self.put_empty_file_name, self.get_empty_file_name)
)
@unittest.expectedFailure
def test_put_with_used_key(self):
self.assertTrue(
self.client.put(
self.put_file_name,
self.bucket_name,
self.key
)
)
@unittest.expectedFailure
def test_get_with_bad_key(self):
self.assertTrue(
self.client.get(
self.get_file_name,
self.bucket_name,
'bad_key'
)
)
def test_get_meta_data(self):
key = random_string.random_string('key_', 12)
put_meta_data = random_string.random_map(5, 8)
self.client.put(
self.put_file_name,
self.bucket_name,
key,
meta_data=put_meta_data
)
get_meta_data = self.client.get_md(
self.bucket_name,
key
)
self.assertTrue(
put_meta_data == get_meta_data
)
import os
def files_are_equal(filename1, filename2):
if not os.path.isfile(filename1):
return False, "Error: File not found: " + filename1
if not os.path.isfile(filename2):
return False, "Error: File not found: " + filename2
file1 = open(filename1)
data1 = file1.read()
file1.close()
file2 = open(filename2)
data2 = file2.read()
file2.close()
if data1 != data2:
return False, "Error: " + filename1 + " and " + filename2 + "do not contain the same data"
return True, ""
import random
import string
def random_string(prefix = '', length = 16):
return prefix + ''.join([random.choice(string.ascii_letters + string.digits) for i in range(length)])
def random_map(size, length):
dictionary = {random_string(): random_string()}
for i in range(size - 1):
dictionary.update({random_string(): random_string()})
return dictionary
import boto3
import uuid
import sys
class S3_client:
def __init__(self, server_url):
self.session = boto3.session.Session()
self.client = self.session.client(
service_name='s3',
endpoint_url=server_url
)
def put(self, filename, bucket_name, key, meta_data = {}):
self.client.upload_file(
Filename = filename,
Bucket = bucket_name,
Key = key,
ExtraArgs={"Metadata": meta_data}
)
def get(self, filename, bucket_name, key):
self.client.download_file(
Filename = filename,
Bucket = bucket_name,
Key = key
)
def get_md(self, bucket_name, key):
response = self.client.head_object(
Bucket = bucket_name,
Key = key
)
return response["Metadata"]
import boto3
import uuid
import sys
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-put', action = 'store_true')
parser.add_argument('-get', action = 'store_true')
parser.add_argument('-getmd', action = 'store_true')
parser.add_argument('-key', dest = 'key_string', default = 'MyKey')
parser.add_argument('-putfile', dest = 'put_file_string', default = 'EmperorWu.txt')
parser.add_argument('-getfile', dest = 'get_file_string', default = 'get_file.txt')
args = parser.parse_args()
def create_bucket_name(bucket_prefix):
# The generated bucket name must be between 3 and 63 chars long
return ''.join([bucket_prefix, str(uuid.uuid4())])
def create_bucket(bucket_prefix, s3_connection):
session = boto3.session.Session()
current_region = session.region_name
bucket_name = create_bucket_name(bucket_prefix)
bucket_response = s3_connection.create_bucket(
Bucket=bucket_name,
CreateBucketConfiguration={
'LocationConstraint': current_region})
print(bucket_name, current_region)
return bucket_name, bucket_response
# s3_resource = boto3.resource('s3')
# s3_client = s3_resource.meta.client
#first_bucket_name, first_response = create_bucket(
#bucket_prefix="ichec.phobos.",
#s3_connection=s3_resource.meta.client)
session = boto3.session.Session()
s3_client = session.client(
service_name='s3',
endpoint_url='http://localhost:11000/'
)
# print("Creating bucket...")
# first_bucket_name, first_response = create_bucket(
# bucket_prefix="ichec.phobos.",
# s3_connection=s3_client)
# print(first_response)
first_bucket_name = "ichec.phobos.7a3d12c6-632b-4291-b87e-07badca60cd0"
if(args.put):
s3_client.upload_file(
Filename=args.put_file_string, Bucket=first_bucket_name,
Key=args.key_string)
if(args.get):
s3_client.download_file(
Filename=args.get_file_string, Bucket=first_bucket_name,
Key=args.key_string)
# if(args.get):
# response = s3_client.get_object(
# Bucket=first_bucket_name,
# Key=args.key_string
# )
# get_file = open(args.get_file_string, "w")
# get_file.write(response["Body"].read())
# This is another command (HEAD request) that will result in a getmd call to phobos!
if(args.getmd):
response = s3_client.head_object(
Bucket=first_bucket_name,
Key=args.key_string
)
print("Meta data:")
print(response["Metadata"])
# response = s3_client.put_object(
# Bucket=first_bucket_name,
# Key=first_file_name,
# Body="smallTestFile.txt",
# Metadata={
# 'metaKey': 'metaVal',
# 'metaKey2': 'metaVal2'
# }
# )
# print("put_object: ")
# print(response)
#print(s3_client.list_buckets())
find_package(Catch2 REQUIRED)
set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test)
add_executable(
tests
......
This diff is collapsed.
......@@ -3,7 +3,7 @@
#include <fstream>
#include <unistd.h>
#include "../src/stream/fifo.h"
#include "../../src/stream/fifo.h"
#include "storage/disk.h"
#include "utils.h"
......
......@@ -4,7 +4,7 @@
#include <fstream>
#include <unistd.h>
#include "../src/storage/phobos_file.h"
#include "../../src/storage/phobos_file.h"
#include "utils.h"
......
#include "../../src/storage/storage.h"
#include "../../../src/storage/storage.h"
namespace fiphoboserver {
namespace storage {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment