diff --git a/gitlamirator/GitHubLab.py b/gitlamirator/GitHubLab.py index f5848a37b11982f1c12108bda3e9ce4f8f8dc96e..2ae38277594eb2ea9a7cc2097fc9aa35265b58e3 100755 --- a/gitlamirator/GitHubLab.py +++ b/gitlamirator/GitHubLab.py @@ -89,7 +89,7 @@ def record_orphan(entry): def cache_gitlab_group(namespace): - """return True when the cache has been updated sucessfully""" + """return True when the cache has been updated successfully""" def get_meta(namespace): try: @@ -155,12 +155,16 @@ def cache_gitlab_group(namespace): return do_update(namespace) -def add_project_to_gitlab(namespace, repoName, repoUrl, issuesEnabledFor=False): +def add_project_to_gitlab(namespace, repoNameRaw, repoUrl, issuesEnabledFor=False): """ Add a mirror project to an *existing* group. The destination namespace should exists beforehand """ + + # Before creating a repository in GitLab we need to sanitize the GitHub repository name to make sure it translate into a valid GitLab repository name + repoName = sanitize_repo_name_for_gitlab(repoNameRaw) + logger.debug("invoke add project {}/{} ...".format(namespace, repoName)) # processing only if the cache is populated with the group data @@ -230,12 +234,19 @@ def add_project_to_gitlab(namespace, repoName, repoUrl, issuesEnabledFor=False): logger.error("Can't update group cache for {}".format(namespace)) +def sanitize_repo_name_for_gitlab(repoName): + # Implementation should be improved to cover more scenarios based on https://docs.gitlab.com/ee/user/reserved_names.html + return repoName.removeprefix('.') + +def sanitize_list_repo_name_for_gitlab(repoNames): + return [sanitize_repo_name_for_gitlab(elem) for elem in repoNames] + # print(repos.text) def is_valid_source(repo, validRepoTypes): """ Determine if the specified repo has special attributes against input list. - if the input list validRepoTypes is ommited and any of the special attributes + if the input list validRepoTypes is omitted and any of the special attributes of ghRepoTypesList is True in the GitHub the repository, then that repo is not considered """ # those are all the possible repo types in GitHub @@ -273,11 +284,8 @@ def print_github_limits(headers): def prepare_repos(doGetSize=False, **itemSettings): - # the dest group is the one read from config if specified otherwise we take - # github's namespace - - # the destination GitLab group could be specified by user in YML. - # If not, fallback to the github org name + # the destination GitLab group could be specified by user in the YAML configuration file. + # If not specified in the configuration, fallback to the GitHub organization name dstGroup = itemSettings['glDestGroup'] if itemSettings['glDestGroup'] else itemSettings['ghNs'] if itemSettings['ghRepo']: @@ -322,6 +330,12 @@ def prepare_repos(doGetSize=False, **itemSettings): else: # a whole namespace + + # If ghRepoOnlyList is defined + if itemSettings['ghRepoOnlyList']: + # Sanitize the GitHub repository names of the "only" setting as we will compare them with GitLab project names + gitlab_repo_only_list = sanitize_list_repo_name_for_gitlab(itemSettings['ghRepoOnlyList']) + repos = sessionGihubApi.get( "https://api.github.com/users/{}/repos?per_page=100".format(itemSettings['ghNs']), timeout=5) if repos.status_code == requests.codes.ok: @@ -362,11 +376,11 @@ def prepare_repos(doGetSize=False, **itemSettings): if cache_gitlab_group(dstGroup): pInGitLab = [p['name'] for p in cacheGroup['projects']] # we sort out if the configured repos are still in GitHub ('only' list) - if itemSettings['ghRepoOnlyList']: - if set(itemSettings['ghRepoOnlyList']) <= set(pInGitLab): - logger.info("all configured repos are in Github") + if gitlab_repo_only_list: + if set(gitlab_repo_only_list) <= set(pInGitLab): + logger.info("all configured repos are in GitHub") - _orphans = set(pInGitLab) - set(itemSettings['ghRepoOnlyList']) + _orphans = set(pInGitLab) - set(gitlab_repo_only_list) if _orphans: logger.warning( 'Warning, "ONLY" list: some of the existing GitLab repos are not specified in the source configuration. See orphan log.') @@ -377,7 +391,7 @@ def prepare_repos(doGetSize=False, **itemSettings): else: # If the whole org is configured for sync # we print out repos that are in GitLab but not in GitHub (anymore) - _reposNotInGH = list(set(pInGitLab) - set(ghRepos.keys())) + _reposNotInGH = list(set(pInGitLab) - set(sanitize_list_repo_name_for_gitlab(ghRepos.keys()))) if _reposNotInGH: _pApiUrls = {p['path_with_namespace']: '{}/projects/{}'.format(config.globalCfg['gitlabApiUrl'], p['id']) for p in cacheGroup['projects'] for orphan in _reposNotInGH if orphan == p['name']} diff --git a/gitlamirator/mirrorIt.py b/gitlamirator/mirrorIt.py index 4d9b4581b2e2ac59018ea460b215dc8830740334..2ca424167179ff1bf0a0dc04e7c214396797050f 100755 --- a/gitlamirator/mirrorIt.py +++ b/gitlamirator/mirrorIt.py @@ -130,8 +130,8 @@ def gen_settings(item): if 'only' in item: logger.error("conflict: can't use 'only' along with specific repository") raise ValueError - else: - # We have a valid setting. We store the name of the repository in gh_repo and the name of the organization/user in gh_ns + else: + #We store the name of the repository in gh_repo and the name of the organization/user in gh_ns gh_ns, gh_repo = src_parts # If we don't have a '/' in the ghsrc value, it means that we want to synchronize a whole GitHub organization / all repositories of a user elif len(src_parts) == 1: @@ -159,7 +159,7 @@ def gen_settings(item): # If we are synchronizing from GitHub if src_type == 'ghsrc': - # Get the organization / user name and optionnaly (if a specific repository is targeted) the repository name + # Get the organization / user name and optionally (if a specific repository is targeted) the repository name gh_ns, gh_repo = get_github_parts(item['ghsrc']) # If we are synchronizing from something else then GitHub else: diff --git a/tests/test_GitHubLab.py b/tests/test_GitHubLab.py new file mode 100644 index 0000000000000000000000000000000000000000..863a7396e7369071609081cbab1282c4c80dc147 --- /dev/null +++ b/tests/test_GitHubLab.py @@ -0,0 +1,15 @@ +#!/usr/bin/python3 + +import unittest +from gitlamirator import GitHubLab + + +class GitHubLabTest(unittest.TestCase): + def test_sanitize_gitlab_repo_name(self): + valid_gitlab_repo_name = 'my-valid-repo-name' + self.assertEqual(GitHubLab.sanitize_repo_name_for_gitlab(valid_gitlab_repo_name), valid_gitlab_repo_name) + + self.assertEqual(GitHubLab.sanitize_repo_name_for_gitlab('.dot'), 'dot') + + # TODO uncomment line below once sanitization improved + #self.assertEqual(GitHubLab.sanitize_repo_name_for_gitlab('@at'), 'at') \ No newline at end of file