Fix pagination, add group filtering option

check_gitlab.py

@@ -12,12 +12,11 @@
 # apt-get install ripgrep
 
 import os
-import re
 import subprocess
 import json
 import csv
 from git import Repo
-from requests import Request, Session
+from requests import Session
 from pathlib import Path
 
 class GitlabConnector:
@@ -54,22 +53,34 @@ class Report():
             for row in self.findings:
                 writer.writerow(row)
 
-def get_all_projects(next_link=None, prev_result=[]):
+def get_all_projects(next_link=None, group_id=None, prev_result=[]):
+    base_path = '/api/v4'
+    url_params = ["include_subgroups=true", "per_page=50", "search_namespaces=true", "owned=false", "order_by=id", "sort=asc"]
+    if group_id:
+        base_path += f"/groups/{group_id}"
+    else:
+        url_params.append("pagination=keyset")
+
     if not next_link:
-        result = session.query("/api/v4/projects?pagination=keyset&per_page=50&search_namespaces=true&owned=false&order_by=id&sort=asc")
+        result = session.query(f"{base_path}/projects?{'&'.join(url_params)}")
     else:
         result = session.get(next_link)
-    
+
     if result.headers.get('Link'):
-        link = result.headers['Link'].split(';')[0].replace('<', '').replace('>', '')
-        rel = result.headers['Link'].split(';')[1].split('=')[1]
+        links = result.headers['Link'].split(', ')
+        for link in links:
+            parts = link.split('; ')
+            rel = parts[1].split('=')[1]
+            if rel == '"next"':
+                link = parts[0].replace('<', '').replace('>', '')
+                break
 
     prev_result += [{'id': i['id'], 'http_url_to_repo': i['http_url_to_repo'], 'ssh_url_to_repo': i['ssh_url_to_repo'], 'web_url': i['web_url']} for i in result.json()]
-    
+
     # I know, not nice.. but im in a hurry
     try:
         if rel == "\"next\"":
-            get_all_projects(next_link=link, prev_result=prev_result)
+            get_all_projects(next_link=link, group_id=group_id, prev_result=prev_result)
     except:
         pass
     return prev_result
@@ -146,7 +157,7 @@ def check_line_in_file(file=None, line_number=None):
 
 
 def check_repos():
-    repos = get_all_projects()
+    repos = get_all_projects(group_id=os.environ.get('GITLAB_GROUP'))
     print(f"Found {len(repos)} Repositories..")
     for repo in repos:
         scan_path = clone_repo_with_http(repo['http_url_to_repo'])
@@ -167,4 +178,4 @@ session = GitlabConnector()
 report = Report()
 check_repos()
 report.results()
-report.write_report(f"{report_path}/{report_file}")
+report.write_report(f"{report_path}/{report_file}")