Formatting for Python integration tests

Commit log:
* add yapf execution to formatting script
* run formatting script
* rename formatting and linting jobs
* install yapf in Format job

.gitlab-ci.yml

@@ -53,7 +53,7 @@ stages:
     - build
     - test
 
-Clang Format:
+Format:
     <<: *default_job
     stage: static analysis
     variables:
@@ -64,13 +64,14 @@ Clang Format:
         - apt-get install -y
             git
             clang-format
+        - pip3 install yapf
     script:
         - ./tools/run_format.sh
             && git status
             && git diff-index --quiet HEAD
             || { echo "This commit contains unformatted files! Run tools/run_format.sh on the project to format them correctly."; false; }
 
-Clang Tidy:
+Lint:
     <<: *default_job
     stage: static analysis
     variables:

test/integration_tests/client.py

@@ -9,12 +9,15 @@ import utils.s3_client as s3_client
 import utils.compare as compare
 import utils.random_string as random_string
 
+
 class Server_test_case(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.bucket_name = "ichec.phobos.7a3d12c6-632b-4291-b87e-07badca60cd0"
         cls.client = s3_client.S3_client('http://localhost:11000/')
-        cls.server = subprocess.Popen('../../build/fiphoboserver', shell=True, preexec_fn=os.setsid)
+        cls.server = subprocess.Popen('../../build/fiphoboserver',
+                                      shell=True,
+                                      preexec_fn=os.setsid)
         cls.put_file_name = 'data/EmperorWu.txt'
         cls.get_file_name = 'data/EmperorWu_get.txt'
         cls.put_empty_file_name = 'data/empty.txt'
@@ -27,101 +30,56 @@ class Server_test_case(unittest.TestCase):
         os.remove(cls.get_empty_file_name)
         cls.server.terminate()
 
-        # kill the fiphoboserver process 
+        # kill the fiphoboserver process
         os.killpg(os.getpgid(cls.server.pid), signal.SIGTERM)
 
     def test_file_put_and_get(self):
-        self.client.put(
-            self.put_file_name,
-            self.bucket_name,
-            self.key 
-        )
-        self.client.get(
-            self.get_file_name,
-            self.bucket_name,
-            self.key
-        )
+        self.client.put(self.put_file_name, self.bucket_name, self.key)
+        self.client.get(self.get_file_name, self.bucket_name, self.key)
         self.assertTrue(
-            compare.files_are_equal(self.put_file_name, self.get_file_name)
-        )
+            compare.files_are_equal(self.put_file_name, self.get_file_name))
+
     def test_empty_file_put_and_get(self):
         key = random_string.random_string('key_', 12)
 
-        self.client.put(
-            self.put_empty_file_name,
-            self.bucket_name,
-            key 
-        )
-        self.client.get(
-            self.get_empty_file_name,
-            self.bucket_name,
-            key
-        )
+        self.client.put(self.put_empty_file_name, self.bucket_name, key)
+        self.client.get(self.get_empty_file_name, self.bucket_name, key)
         self.assertTrue(
-            compare.files_are_equal(self.put_empty_file_name, self.get_empty_file_name)
-        )
+            compare.files_are_equal(self.put_empty_file_name,
+                                    self.get_empty_file_name))
 
     @unittest.expectedFailure
     def test_put_with_used_key(self):
         self.assertTrue(
-            self.client.put(
-                self.put_file_name,
-                self.bucket_name,
-                self.key 
-            )
-        )
+            self.client.put(self.put_file_name, self.bucket_name, self.key))
 
     @unittest.expectedFailure
     def test_get_with_bad_key(self):
         self.assertTrue(
-            self.client.get(
-                self.get_file_name,
-                self.bucket_name,
-                'bad_key'
-            )
-        )
+            self.client.get(self.get_file_name, self.bucket_name, 'bad_key'))
 
     def test_get_meta_data(self):
         key = random_string.random_string('key_', 12)
         put_meta_data = random_string.random_map(5, 8)
 
-        self.client.put(
-            self.put_file_name,
-            self.bucket_name,
-            key,
-            meta_data=put_meta_data
-        )
+        self.client.put(self.put_file_name,
+                        self.bucket_name,
+                        key,
+                        meta_data=put_meta_data)
 
-        get_meta_data = self.client.get_md(
-            self.bucket_name,
-            key
-        )
+        get_meta_data = self.client.get_md(self.bucket_name, key)
 
-        self.assertTrue(
-            put_meta_data == get_meta_data
-        )
+        self.assertTrue(put_meta_data == get_meta_data)
 
     def client_process(self):
         key = random_string.random_string('key_', 12)
         get_file_name = random_string.random_string('data/', 11)
 
-        self.client.put(
-            self.put_file_name,
-            self.bucket_name,
-            key 
-        )
-        self.client.get(
-            get_file_name,
-            self.bucket_name,
-            key
-        )
-        get_meta_data = self.client.get_md(
-            self.bucket_name,
-            key
-        )
+        self.client.put(self.put_file_name, self.bucket_name, key)
+        self.client.get(get_file_name, self.bucket_name, key)
+        get_meta_data = self.client.get_md(self.bucket_name, key)
         self.assertTrue(
-            compare.files_are_equal(self.put_file_name, get_file_name)
-        )
+            compare.files_are_equal(self.put_file_name, get_file_name))
 
         # clean up files
         os.remove(get_file_name)

test/integration_tests/utils/compare.py

 import os
 
+
 def files_are_equal(filename1, filename2):
     if not os.path.isfile(filename1):
         return False, "Error: File not found: " + filename1

test/integration_tests/utils/random_string.py

 import random
 import string
 
-def random_string(prefix = '', length = 16):
-    return prefix + ''.join([random.choice(string.ascii_letters + string.digits) for i in range(length)])
+
+def random_string(prefix='', length=16):
+    return prefix + ''.join([
+        random.choice(string.ascii_letters + string.digits)
+        for i in range(length)
+    ])
+
 
 def random_map(size, length):
     dictionary = {random_string(): random_string()}

test/integration_tests/utils/s3_client.py

@@ -2,33 +2,24 @@ import boto3
 import uuid
 import sys
 
+
 class S3_client:
     def __init__(self, server_url):
         self.session = boto3.session.Session()
-        self.client = self.session.client(
-            service_name='s3',
-            endpoint_url=server_url
-        )
+        self.client = self.session.client(service_name='s3',
+                                          endpoint_url=server_url)
 
-    def put(self, filename, bucket_name, key, meta_data = {}):
-        self.client.upload_file(
-            Filename = filename,
-            Bucket = bucket_name,
-            Key = key,
-            ExtraArgs={"Metadata": meta_data}
-        )
+    def put(self, filename, bucket_name, key, meta_data={}):
+        self.client.upload_file(Filename=filename,
+                                Bucket=bucket_name,
+                                Key=key,
+                                ExtraArgs={"Metadata": meta_data})
 
     def get(self, filename, bucket_name, key):
-        self.client.download_file(
-            Filename = filename,
-            Bucket = bucket_name,
-            Key = key
-        )
+        self.client.download_file(Filename=filename,
+                                  Bucket=bucket_name,
+                                  Key=key)
 
     def get_md(self, bucket_name, key):
-        response = self.client.head_object(
-            Bucket = bucket_name,
-            Key = key
-        )
+        response = self.client.head_object(Bucket=bucket_name, Key=key)
         return response["Metadata"]
-

tools/run_format.sh

@@ -33,4 +33,8 @@ do
                 | xargs -n 1 -0 -P "${nprocs}" ${FORMAT_EXECUTABLE}
         fi
     done
+    if [ ! -z "$(find $dir -iname "*.py")" ]
+    then
+        yapf -ir $dir
+    fi
 done