Skip to content

Commit

Permalink
Merge branch 'main' into project-artifacts-2
Browse files Browse the repository at this point in the history
  • Loading branch information
wy65701436 authored Aug 2, 2024
2 parents 0e428d4 + 295e075 commit 482eb9c
Show file tree
Hide file tree
Showing 12 changed files with 359 additions and 9 deletions.
25 changes: 25 additions & 0 deletions tests/apitests/python/library/artifact.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ def get_reference_info(self, project_name, repo_name, reference, expect_status_c
if "with_scan_overview" in kwargs:
params["with_scan_overview"] = kwargs["with_scan_overview"]
params["x_accept_vulnerabilities"] = ",".join(report_mime_types)
if "with_sbom_overview" in kwargs:
params["with_sbom_overview"] = kwargs["with_sbom_overview"]
if "with_immutable_status" in kwargs:
params["with_immutable_status"] = kwargs["with_immutable_status"]
if "with_accessory" in kwargs:
Expand Down Expand Up @@ -140,6 +142,29 @@ def check_image_scan_result(self, project_name, repo_name, reference, expected_s
return
raise Exception("Scan image result is {}, not as expected {}.".format(scan_status, expected_scan_status))

def check_image_sbom_generation_result(self, project_name, repo_name, reference, expected_scan_status = "Success", **kwargs):
timeout_count = 30
scan_status=""
while True:
time.sleep(5)
timeout_count = timeout_count - 1
if (timeout_count == 0):
break
artifact = self.get_reference_info(project_name, repo_name, reference, **kwargs)
if expected_scan_status in ["Not Scanned", "No SBOM Overview"]:
if artifact.sbom_overview is None:
if (timeout_count > 24):
continue
print("artifact SBOM is not generated.")
return
else:
raise Exception("Artifact SBOM should not be generated {}.".format(artifact.sbom_overview))

scan_status = artifact.sbom_overview.scan_status
if scan_status == expected_scan_status:
return
raise Exception("Generate image SBOM result is {}, not as expected {}.".format(scan_status, expected_scan_status))

def check_reference_exist(self, project_name, repo_name, reference, ignore_not_found = False, **kwargs):
artifact = self.get_reference_info( project_name, repo_name, reference, ignore_not_found=ignore_not_found, **kwargs)
return {
Expand Down
15 changes: 15 additions & 0 deletions tests/apitests/python/library/scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,18 @@ def scan_artifact(self, project_name, repo_name, reference, expect_status_code =
base._assert_status_code(expect_status_code, status_code)

return data

def sbom_generation_of_artifact(self, project_name, repo_name, reference, expect_status_code = 202, expect_response_body = None, **kwargs):
try:
req_param = dict(scan_type = {"scan_type":"sbom"})
data, status_code, _ = self._get_client(**kwargs).scan_artifact_with_http_info(project_name, repo_name, reference, **req_param)
except ApiException as e:
base._assert_status_code(expect_status_code, e.status)
if expect_response_body is not None:
base._assert_status_body(expect_response_body, e.body)
return

base._assert_status_code(expect_status_code, status_code)

return data

15 changes: 15 additions & 0 deletions tests/apitests/python/library/scan_stop.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,19 @@ def stop_scan_artifact(self, project_name, repo_name, reference, expect_status_c

base._assert_status_code(expect_status_code, status_code)

return data

def stop_sbom_generation_of_artifact(self, project_name, repo_name, reference, expect_status_code = 202, expect_response_body = None, **kwargs):
try:
scanType = v2_swagger_client.ScanType()
scanType.scan_type = "sbom"
data, status_code, _ = self._get_client(**kwargs).stop_scan_artifact_with_http_info(project_name, repo_name, reference, scanType)
except ApiException as e:
base._assert_status_code(expect_status_code, e.status)
if expect_response_body is not None:
base._assert_status_body(expect_response_body, e.body)
return

base._assert_status_code(expect_status_code, status_code)

return data
15 changes: 10 additions & 5 deletions tests/apitests/python/test_project_permission.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def call(self):
read_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "GET", 200, metadata_payload)
metadata_payload_for_update = { "auto_scan": "false" }
update_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "PUT", 200, metadata_payload_for_update)
delete_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "DELETE", 200, metadata_payload)
delete_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "DELETE", 200, metadata_payload_for_update)

# 4. Resource: repository actions: ['read', 'list', 'update', 'delete', 'pull', 'push']
# note: pull and push are for docker cli, no API needs them
Expand All @@ -89,12 +89,17 @@ def call(self):
delete_artifact = Permission("{}/projects/{}/repositories/target_repo/artifacts/{}".format(harbor_base_url, project_name, source_artifact_tag), "DELETE", 200)

# 6. Resource scan actions: ['read', 'create', 'stop']
stop_scan_payload = {
vulnerability_scan_payload = {
"scan_type": "vulnerability"
}
create_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202)
stop_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, stop_scan_payload)
create_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, vulnerability_scan_payload)
stop_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, vulnerability_scan_payload)
read_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/83be44fd-1234-5678-b49f-4b6d6e8f5730/log".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "get", 404)
sbom_gen_payload = {
"scan_type": "sbom"
}
create_sbom_generation = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, sbom_gen_payload)
stop_sbom_generation = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, sbom_gen_payload)

# 7. Resource tag actions: ['list', 'create', 'delete']
tag_payload = { "name": "test-{}".format(int(random.randint(1000, 9999))) }
Expand Down Expand Up @@ -240,7 +245,7 @@ def call(self):
"metadata": [create_metadata, list_metadata, read_metadata, update_metadata, delete_metadata],
"repository": [list_repo, read_repo, update_repo, delete_repo],
"artifact": [list_artifact, read_artifact, copy_artifact, delete_artifact],
"scan": [create_scan, stop_scan, read_scan],
"scan": [create_scan, stop_scan, read_scan, create_sbom_generation, stop_sbom_generation],
"tag": [create_tag, list_tag, delete_tag],
"accessory": [list_accessory],
"artifact-addition": [read_artifact_addition_vul, read_artifact_addition_dependencies],
Expand Down
87 changes: 87 additions & 0 deletions tests/apitests/python/test_sbom_generation_of_image_artifact.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
from __future__ import absolute_import
import unittest
import sys

from testutils import harbor_server, suppress_urllib3_warning
from testutils import TEARDOWN
from testutils import ADMIN_CLIENT, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME
from library.project import Project
from library.user import User
from library.repository import Repository
from library.repository import push_self_build_image_to_project
from library.artifact import Artifact
from library.scan import Scan

class TestSBOMGeneration(unittest.TestCase):
@suppress_urllib3_warning
def setUp(self):
self.project= Project()
self.user= User()
self.artifact = Artifact()
self.repo = Repository()
self.scan = Scan()

self.url = ADMIN_CLIENT["endpoint"]
self.user_password = "Aa123456"
self.project_id, self.project_name, self.user_id, self.user_name, self.repo_name1 = [None] * 5
self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT)
self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password, with_sbom_overview = True)


#2. Create a new private project(PA) by user(UA);
self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT)

#3. Add user(UA) as a member of project(PA) with project-admin role;
self.project.add_project_members(self.project_id, user_id = self.user_id, **ADMIN_CLIENT)

@unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
def do_tearDown(self):
#1. Delete repository(RA) by user(UA);
self.repo.delete_repository(self.project_name, self.repo_name1.split('/')[1], **self.USER_CLIENT)

#2. Delete project(PA);
self.project.delete_project(self.project_id, **self.USER_CLIENT)

#3. Delete user(UA);
self.user.delete_user(self.user_id, **ADMIN_CLIENT)

def testGenerateSBOMOfImageArtifact(self):
"""
Test case:
Generate an SBOM of An Image Artifact
Test step and expected result:
1. Create a new user(UA);
2. Create a new private project(PA) by user(UA);
3. Add user(UA) as a member of project(PA) with project-admin role;
4. Get private project of user(UA), user(UA) can see only one private project which is project(PA);
5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
6. Send sbom generation of an image command and get tag(TA) information to check sbom generation result, it should be finished;
Tear down:
1. Delete repository(RA) by user(UA);
2. Delete project(PA);
3. Delete user(UA);
"""

#4. Get private project of user(UA), user(UA) can see only one private project which is project(PA);
self.project.projects_should_exist(dict(public=False), expected_count = 1,
expected_project_id = self.project_id, **self.USER_CLIENT)

#Note: Please make sure that this Image has never been pulled before by any other cases,
# so it is a not-scanned image right after repository creation.
image = "docker"
src_tag = "1.13"
#5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
self.repo_name1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag)

#6. Send sbom generation of an image command and get tag(TA) information to check sbom generation result, it should be finished;
self.scan.sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT)
self.artifact.check_image_sbom_generation_result(self.project_name, image, tag, **self.USER_CLIENT)

self.do_tearDown()


if __name__ == '__main__':
suite = unittest.TestSuite(unittest.makeSuite(TestSBOMGeneration))
result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite)
if not result.wasSuccessful():
raise Exception(r"SBOM generation test failed: {}".format(result))
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
from __future__ import absolute_import
import unittest
import sys

from testutils import harbor_server, suppress_urllib3_warning
from testutils import TEARDOWN
from testutils import ADMIN_CLIENT, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME
from library.project import Project
from library.user import User
from library.repository import Repository
from library.repository import push_self_build_image_to_project
from library.artifact import Artifact
from library.scan import Scan
from library.scan_stop import StopScan

class TestStopSBOMGeneration(unittest.TestCase):
@suppress_urllib3_warning
def setUp(self):
self.project= Project()
self.user= User()
self.artifact = Artifact()
self.repo = Repository()
self.scan = Scan()
self.stop_scan = StopScan()

self.url = ADMIN_CLIENT["endpoint"]
self.user_password = "Aa123456"
self.project_id, self.project_name, self.user_id, self.user_name, self.repo_name1 = [None] * 5
self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT)
self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password, with_sbom_overview = True)


#2. Create a new private project(PA) by user(UA);
self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT)

#3. Add user(UA) as a member of project(PA) with project-admin role;
self.project.add_project_members(self.project_id, user_id = self.user_id, **ADMIN_CLIENT)

@unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
def do_tearDown(self):
#1. Delete repository(RA) by user(UA);
self.repo.delete_repository(self.project_name, self.repo_name1.split('/')[1], **self.USER_CLIENT)

#2. Delete project(PA);
self.project.delete_project(self.project_id, **self.USER_CLIENT)

#3. Delete user(UA);
self.user.delete_user(self.user_id, **ADMIN_CLIENT)

def testStopSBOMGenerationOfImageArtifact(self):
"""
Test case:
Stop SBOM Generation Of An Image Artifact
Test step and expected result:
1. Create a new user(UA);
2. Create a new private project(PA) by user(UA);
3. Add user(UA) as a member of project(PA) with project-admin role;
4. Get private project of user(UA), user(UA) can see only one private project which is project(PA);
5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
6. Send SBOM generation of an image command;
7. Send stop SBOM generation of an image command.
Tear down:
1. Delete repository(RA) by user(UA);
2. Delete project(PA);
3. Delete user(UA);
"""

#4. Get private project of user(UA), user(UA) can see only one private project which is project(PA);
self.project.projects_should_exist(dict(public=False), expected_count = 1,
expected_project_id = self.project_id, **self.USER_CLIENT)

#Note: Please make sure that this Image has never been pulled before by any other cases,
# so it is a not-scanned image right after repository creation.
image = "docker"
src_tag = "1.13"
#5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
self.repo_name1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag)

#6. Send SBOM generation of an image command;
self.scan.sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT)

#7. Send stop SBOM generation of an image command.
self.stop_scan.stop_sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT)

self.do_tearDown()

if __name__ == '__main__':
suite = unittest.TestSuite(unittest.makeSuite(TestStopSBOMGeneration))
result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite)
if not result.wasSuccessful():
raise Exception(r"Stop SBOM generation test failed: {}".format(result))
17 changes: 14 additions & 3 deletions tests/resources/Harbor-Pages/Project-Repository.robot
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,20 @@ Stop Scan Artifact
Retry Element Click ${stop_scan_artifact_btn}

Check Scan Artifact Job Status Is Stopped
Wait Until Element Is Visible ${stopped_label}
${job_status}= Get Text ${stopped_label}
Should Be Equal As Strings '${job_status}' 'Scan stopped'
Wait Until Element Is Visible ${scan_stopped_label}

Generate Artifact SBOM
[Arguments] ${project} ${repo} ${label_xpath}=//clr-dg-row//label[contains(@class,'clr-control-label')][1]
Go Into Repo ${project} ${repo}
Retry Element Click ${label_xpath}
Retry Element Click ${gen_artifact_sbom_btn}

Stop Gen Artifact SBOM
Retry Element Click ${artifact_action_xpath}
Retry Element Click ${stop_gen_artifact_sbom_btn}

Check Gen Artifact SBOM Job Status Is Stopped
Wait Until Element Is Visible ${gen_sbom_stopped_label}

Refresh Repositories
Retry Element Click ${refresh_repositories_xpath}
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,8 @@ ${build_history_data} //clr-dg-row
${push_image_command_btn} //hbr-push-image-button//button
${scan_artifact_btn} //button[@id='scan-btn']
${stop_scan_artifact_btn} //button[@id='stop-scan']
${stopped_label} //span[@class='label stopped']
${scan_stopped_label} //span[normalize-space()='Scan stopped']
${gen_sbom_stopped_label} //span[normalize-space()='Generation stopped']
${gen_artifact_sbom_btn} //button[@id='generate-sbom-btn']
${stop_gen_artifact_sbom_btn} //button[@id='stop-sbom-btn']
${refresh_repositories_xpath} //hbr-repository-gridview//span[contains(@class,'refresh-btn')]
35 changes: 35 additions & 0 deletions tests/resources/Harbor-Pages/Vulnerability.robot
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,41 @@ Enable Scan On Push
Checkbox Should Be Selected //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input
Retry Element Click ${project_config_save_btn}

Generate Repo SBOM
[Arguments] ${tagname} ${status}
Retry Element Click //clr-dg-row[contains(.,'${tagname}')]//label[contains(@class,'clr-control-label')]
Retry Element Click //button[@id='generate-sbom-btn']
Run Keyword If '${status}' == 'Succeed' Wait Until Element Is Visible //a[@title='SBOM details'] 300

Checkout And Review SBOM Details
[Arguments] ${tagname}
Retry Element Click //clr-dg-row[contains(.,'${tagname}')]//a[@title='SBOM details']
# Download SBOM file
Retry Element Click //button[@id='sbom-btn']
${sbom_artifact_short_sha256}= Get Text //span[@class='margin-left-10px']
${sbom_filename_raw}= Get Text //clr-dg-cell[contains(text(),'${sbom_artifact_short_sha256}')]
${sbom_filename}= Replace String ${sbom_filename_raw} : _ count=-1
${sbom_filename}= Replace String ${sbom_filename} / _ count=-1
${sbom_json_path}= Set Variable ${download_directory}/${sbom_filename}.json
Retry File Should Exist ${sbom_json_path}
# Load the downloaded SBOM json file and verify the first N package records
${sbom_json_content}= Load Json From File ${sbom_json_path}
${items}= Get Value From JSON ${sbom_json_content} packages
${items_length}= Get Length ${items}
${first_n_records}= Evaluate min(5, ${items_length})
FOR ${idx} IN RANGE 1 ${first_n_records}
${item}= Get From List ${items} ${idx}
Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.name}']
Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.versionInfo}']
Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.licenseConcluded}']
END

Enable Generating SBOM On Push
Checkbox Should Not Be Selected //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//input
Retry Element Click //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//label[contains(@class,'clr-control-label')]
Checkbox Should Be Selected //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//input
Retry Element Click ${project_config_save_btn}

Vulnerability Not Ready Project Hint
Sleep 2
${element}= Set Variable xpath=//span[contains(@class, 'db-status-warning')]
Expand Down
Loading

0 comments on commit 482eb9c

Please sign in to comment.