199 lines
8.0 KiB
Python
Executable File
199 lines
8.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
""" A script to find github repo links and last commit dates in a markdown file
|
|
|
|
Requirements:
|
|
- python3 github module (sudo apt install python3-github on Debian)
|
|
- A personal access token (https://github.com/settings/tokens)
|
|
|
|
Usage:
|
|
- Github graphql API calls are limited to 5000 points/hour https://docs.github.com/en/graphql/overview/resource-limitations
|
|
- Put the token in your environment variables:
|
|
export GITHUB_TOKEN=18c45f8d8d556492d1d877998a5b311b368a76e4
|
|
- The output is sorted oldest to newest
|
|
- Put the script in your crontab or run it from time to time. It doesn't make sense to add this
|
|
script to the CI job that runs every time something is pushed.
|
|
- To detect no-commit related activity (repo metadata changes, wiki edits, ...), replace pushed_at
|
|
with updated_at
|
|
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
import re
|
|
import os
|
|
import logging
|
|
import requests
|
|
from requests.adapters import HTTPAdapter
|
|
from requests.exceptions import ConnectionError
|
|
from datetime import *
|
|
|
|
__author__ = "nodiscc"
|
|
__copyright__ = "Copyright 2019, nodiscc"
|
|
__credits__ = ["https://github.com/awesome-selfhosted/awesome-selfhosted"]
|
|
__license__ = "MIT"
|
|
__version__ = "1.0"
|
|
__maintainer__ = "nodiscc"
|
|
__email__ = "nodiscc@gmail.com"
|
|
__status__ = "Production"
|
|
|
|
###############################################################################
|
|
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
|
|
|
|
""" function to query Github graphql API """
|
|
def query_github_api(query, variables):
|
|
access_token = os.environ['GITHUB_TOKEN']
|
|
headers = {"Authorization": "Bearer " + access_token}
|
|
github_adapter = HTTPAdapter(max_retries=7)
|
|
session = requests.Session()
|
|
session.mount('https:api.github.com/graphql', github_adapter)
|
|
try:
|
|
logging.info('Querying API for %s', variables)
|
|
response = session.post('https://api.github.com/graphql', timeout=(10) , json={'query': query, 'variables': variables}, headers=headers)
|
|
response.raise_for_status()
|
|
logging.debug(response.json())
|
|
return response.json()
|
|
except requests.exceptions.HTTPError as errh:
|
|
logging.error("An Http Error occurred:" + repr(errh))
|
|
return {'errors': [{'type': 'HTTP Error'}]}
|
|
except requests.exceptions.ConnectionError as errc:
|
|
logging.error("An Error Connecting to the API occurred:" + repr(errc))
|
|
return {"errors": [ { "type": "Connect Error"}]}
|
|
except requests.exceptions.Timeout as errt:
|
|
logging.error("A Timeout Error occurred:" + repr(errt))
|
|
return {"errors": [ { "type": "Timeout Error"}]}
|
|
except requests.exceptions.RequestException as err:
|
|
logging.error("An Unknown Error occurred" + repr(err))
|
|
return {"errors": [ { "type": "Request Exception"}]}
|
|
|
|
""" function to add commas for prettier output"""
|
|
def add_comma(s):
|
|
if s != '':
|
|
s = ', ' + s
|
|
return s
|
|
else:
|
|
return s
|
|
|
|
output = []
|
|
|
|
""" find all URLs of the form https://github.com/owner/repo """
|
|
def parse_github_projects():
|
|
with open(sys.argv[1], 'r') as readme:
|
|
logging.info('Testing ' + sys.argv[1])
|
|
data = readme.read()
|
|
#project_urls = re.findall('https://github\.com/.*', data)
|
|
project_urls = re.findall('https://github\.com/([a-zA-Z\d\-\._]{1,39}/[a-zA-Z\d\-\._]{1,39})(?=\)|/|#\s)', data)
|
|
logging.info('Checking ' + str(len(project_urls)) + ' github repos.')
|
|
return sorted(set(project_urls))
|
|
|
|
urls = parse_github_projects()
|
|
|
|
""" function to check remaining rate limit """
|
|
def check_github_remaining_limit():
|
|
query = '''
|
|
query{
|
|
viewer {
|
|
login
|
|
}
|
|
rateLimit {
|
|
cost
|
|
remaining
|
|
resetAt
|
|
}
|
|
}'''
|
|
logging.info("Checking github api remaining rate limit.")
|
|
result = query_github_api(query, '')
|
|
if 'errors' in result:
|
|
logging.error(result["errors"][0]["type"] + ", " + result["errors"][0]["message"])
|
|
with open('github_commit_dates.md', 'w') as filehandle:
|
|
filehandle.write('%s\n' % '--------------------\n### Github commit date checks')
|
|
filehandle.write(result["errors"][0]["type"] + ", " + result["errors"][0]["message"])
|
|
else:
|
|
if result["data"]["rateLimit"]["remaining"] < len(urls):
|
|
logging.error('Github api calls remaining is insufficient, exiting.')
|
|
logging.error('URLS: ' + str(len(urls)) + ', api calls remaining: ' + str(result["data"]["rateLimit"]["remaining"]) + ', Resets at: ' + str(result["data"]["rateLimit"]["resetAt"]))
|
|
with open('github_commit_dates.md', 'w') as filehandle:
|
|
filehandle.write('%s\n' % '--------------------\n### Github commit date checks')
|
|
filehandle.write('Github api calls remaining is insufficient, exiting.\n')
|
|
filehandle.write('URLS: ' + str(len(urls)) + ', api calls remaining: ' + str(result["data"]["rateLimit"]["remaining"]) + ', Resets at: ' + str(result["data"]["rateLimit"]["resetAt"]) + '\n')
|
|
sys.exit(1)
|
|
|
|
""" Uncomment this to debug the list of matched URLs """
|
|
# print(str(urls))
|
|
# print(len(urls))
|
|
# with open('links.txt', 'w') as filehandle:
|
|
# for l in urls:
|
|
# filehandle.write('%s\n' % l)
|
|
|
|
# exit(0)
|
|
|
|
check_github_remaining_limit()
|
|
i = 0
|
|
""" load project metadata, output last commit date and URL """
|
|
for url in urls:
|
|
split = url.split("/")
|
|
variables = {
|
|
"owner": split[0],
|
|
"name": split[1]
|
|
}
|
|
query = '''
|
|
query($owner: String!, $name: String!){
|
|
repository(owner:$owner, name:$name) {
|
|
pushedAt
|
|
updatedAt
|
|
isArchived
|
|
isDisabled
|
|
nameWithOwner
|
|
}
|
|
rateLimit {
|
|
cost
|
|
remaining
|
|
resetAt
|
|
}
|
|
}'''
|
|
|
|
github_graphql_data = query_github_api(query, variables)
|
|
if 'errors' in github_graphql_data:
|
|
logging.info(github_graphql_data["errors"][0]["type"])
|
|
output.append([date(1900, 1, 1),'https://github.com/'+url, github_graphql_data["errors"][0]["type"]])
|
|
else:
|
|
has_issue = False
|
|
note = ''
|
|
if github_graphql_data["data"]["repository"]["isArchived"] == True:
|
|
has_issue = True
|
|
note = 'Archived'
|
|
if github_graphql_data["data"]["repository"]["isDisabled"] == True:
|
|
if note == '':
|
|
has_issue = True
|
|
note = 'Disabled'
|
|
else:
|
|
note = note + ', Disabled'
|
|
if github_graphql_data["data"]["repository"]["nameWithOwner"] != url:
|
|
if note == '':
|
|
has_issue = True
|
|
note = 'Moved to https://github.com/'+ github_graphql_data["data"]["repository"]["nameWithOwner"]
|
|
else:
|
|
note = note + ', Moved to https://github.com/'+ github_graphql_data["data"]["repository"]["nameWithOwner"]
|
|
project_pushed_at = datetime.strptime(github_graphql_data["data"]["repository"]["pushedAt"], '%Y-%m-%dT%H:%M:%SZ').date()
|
|
if project_pushed_at < (date.today() - timedelta(days = 365)):
|
|
has_issue = True
|
|
if has_issue:
|
|
output.append([project_pushed_at, 'https://github.com/'+url, note])
|
|
logging.info(str(project_pushed_at)+' | https://github.com/'+url+' | '+note)
|
|
i += 1
|
|
|
|
|
|
if i > 0:
|
|
sorted_list = sorted(output, key=lambda x: x[0])
|
|
with open('github_commit_dates.md', 'w') as filehandle:
|
|
filehandle.write('%s\n' % '--------------------\n### Github commit date checks')
|
|
filehandle.write('%s\n' % '#### There were %s repos last updated over 1 year ago.' % str(i))
|
|
for l in sorted_list:
|
|
filehandle.write('* [ ] %s, %s%s \n' % (str(l[0]), l[1], add_comma(l[2])))
|
|
sys.exit(0)
|
|
else:
|
|
with open('github_commit_dates.md', 'w') as filehandle:
|
|
filehandle.write('%s\n' % '--------------------\n### Github commit date checks')
|
|
filehandle.write('%s\n' % '#### There were no repos last updated over 1 year ago.')
|
|
|