From 19937e41db9c98aed0c9cedfdb000c6d4891db3f Mon Sep 17 00:00:00 2001 From: Nathan Rew Date: Tue, 29 Jun 2021 07:57:56 -0500 Subject: [PATCH 1/6] Add github actions workflows Signed-off-by: Nathan Rew --- .github/workflows/monthly.yml | 108 ++++++++++++++++++ .github/workflows/pull_request.yml | 36 ++++++ .travis.yml | 20 ---- tests/check-github-commit-dates.py | 172 ++++++++++++++++++++++++++--- tests/link_whitelist.txt | 1 + tests/test.js | 57 +++++----- 6 files changed, 331 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/monthly.yml create mode 100644 .github/workflows/pull_request.yml delete mode 100644 .travis.yml create mode 100644 tests/link_whitelist.txt diff --git a/.github/workflows/monthly.yml b/.github/workflows/monthly.yml new file mode 100644 index 00000000..7de13091 --- /dev/null +++ b/.github/workflows/monthly.yml @@ -0,0 +1,108 @@ +name: Monthly Checks + +on: + schedule: + - cron: '* * 1 * *' + workflow_dispatch: + +jobs: + create-issue: + if: always() + needs: [check_syntax, check_links, check_github_commit_dates] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + with: + name: result + - name: Create Issue template + run: | + printf '%s\n%s%s %s\n%s\n%s\n' '---' 'title: Monthly Checks - ' $( date +"%B %Y" ) 'labels: automated issue' '---' > .github/ISSUE_TEMPLATE.md + echo -e '[![Monthly Checks](https://github.com/n8225/awesome-selfhosted/actions/workflows/monthly.yml/badge.svg)](https://github.com/n8225/awesome-selfhosted/actions/workflows/monthly.yml)' >> .github/ISSUE_TEMPLATE.md + echo -e '\n--------------------' >> .github/ISSUE_TEMPLATE.md + echo -e '\n### Awersome_Bot link checks\n' >> .github/ISSUE_TEMPLATE.md + jq -r '.[] | ["* [ ] ", "Line ", .loc, ": ", .link, ", ", .error] | join("")' ab-results-README.md-filtered.json >> .github/ISSUE_TEMPLATE.md || true + cat github_commit_dates.md >> .github/ISSUE_TEMPLATE.md || true + cat syntax_check.md >> .github/ISSUE_TEMPLATE.md || true + echo -e '\n--------------------' >> .github/ISSUE_TEMPLATE.md + printf '%s/%s%s%s' ${GITHUB_SERVER_URL} ${GITHUB_REPOSITORY} '/actions/runs/' ${GITHUB_RUN_ID} >> .github/ISSUE_TEMPLATE.md + - name: Verify template + run: cat .github/ISSUE_TEMPLATE.md + - name: create issue + id: create-iss + uses: buluma/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: 'echo Created issue number ${{ steps.create-iss.outputs.number }}' + + check_github_commit_dates: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Setup Checks + run: pip3 install Requests + - name: Checks + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python tests/check-github-commit-dates.py README.md + - name: Check result + if: ${{ always() }} + run: cat github_commit_dates.md + - name: Upload result + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: result + path: github_commit_dates.md + + check_syntax: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v1 + with: + node-version: '14.x' + - name: Setup Checks + run: | + cd tests + npm install --silent chalk + cd .. + - name: Checks + run: + script -e -c 'node tests/test.js -r README.md' + - name: Check result + if: ${{ always() }} + run: cat syntax_check.md + - name: upload check syntax results + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: result + path: syntax_check.md + + check_links: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby 2.6 + uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.6.7 + - name: Setup Checks + run: gem install awesome_bot + - name: Checks + run: awesome_bot -f README.md --allow-redirect --allow 202,429 --white-list < tests/link_whitelist.txt + - name: Check result + if: ${{ always() }} + run: cat ab-results-README.md-filtered.json + - name: upload awesome_bot results + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: result + path: ab-results-*.json diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 00000000..5152a8e7 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,36 @@ +name: Pull Request Checks + +on: + pull_request: + branches: [ main ] + +jobs: + check_syntax: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v1 + with: + node-version: '14.x' + - name: Checks + run: | + cd test + npm install chalk + cd .. + git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md + script -e -c 'node tests/test.js -r README.md -d temp.md' + + check_links: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby 2.6 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.6.x + - name: Checks + run: | + gem install awesome_bot + awesome_bot -f temp.md --allow-redirect --skip-save-results --allow 202 --white-list < tests/awesomebot_whitelist.txt + \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1259a4f5..00000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -language: node_js - -node_js: - - "node" - -cache: - npm: false - -before_install: - - rvm install 2.6.2 - - gem install awesome_bot - - sudo apt update && sudo apt install python3-pip python3-setuptools - - cd tests && npm install chalk && cd .. - -script: - - 'if [[ "$TRAVIS_BRANCH" == "master" && "$TRAVIS_EVENT_TYPE" == "cron" ]]; then make check_all; fi' - - 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then make check_pr; fi' - -notifications: - email: false diff --git a/tests/check-github-commit-dates.py b/tests/check-github-commit-dates.py index 4070700a..f88a2034 100755 --- a/tests/check-github-commit-dates.py +++ b/tests/check-github-commit-dates.py @@ -7,12 +7,10 @@ Requirements: - A personal access token (https://github.com/settings/tokens) Usage: - - Run awesome_bot --allow-redirect -f README.md beforehand to detect any error(4xx, 5xx) that would - cause the script to abort - - Github API calls are limited to 5000 requests/hour https://developer.github.com/v3/#rate-limiting + - Github graphql API calls are limited to 5000 points/hour https://docs.github.com/en/graphql/overview/resource-limitations - Put the token in your environment variables: export GITHUB_TOKEN=18c45f8d8d556492d1d877998a5b311b368a76e4 - - The output is unsorted, just pipe it through 'sort' or paste it in your editor and sort from there + - The output is sorted oldest to newest - Put the script in your crontab or run it from time to time. It doesn't make sense to add this script to the CI job that runs every time something is pushed. - To detect no-commit related activity (repo metadata changes, wiki edits, ...), replace pushed_at @@ -20,11 +18,15 @@ Usage: """ -from github import Github +import json import sys -import time import re import os +import logging +import requests +from requests.adapters import HTTPAdapter +from requests.exceptions import ConnectionError +from datetime import * __author__ = "nodiscc" __copyright__ = "Copyright 2019, nodiscc" @@ -36,25 +38,161 @@ __email__ = "nodiscc@gmail.com" __status__ = "Production" ############################################################################### +logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S') -access_token = os.environ['GITHUB_TOKEN'] +""" function to query Github graphql API """ +def query_github_api(query, variables): + access_token = os.environ['GITHUB_TOKEN'] + headers = {"Authorization": "Bearer " + access_token} + github_adapter = HTTPAdapter(max_retries=7) + session = requests.Session() + session.mount('https:api.github.com/graphql', github_adapter) + try: + logging.info('Querying API for %s', variables) + response = session.post('https://api.github.com/graphql', timeout=(10) , json={'query': query, 'variables': variables}, headers=headers) + response.raise_for_status() + logging.debug(response.json()) + return response.json() + except requests.exceptions.HTTPError as errh: + logging.error("An Http Error occurred:" + repr(errh)) + return {'errors': [{'type': 'HTTP Error'}]} + except requests.exceptions.ConnectionError as errc: + logging.error("An Error Connecting to the API occurred:" + repr(errc)) + return {"errors": [ { "type": "Connect Error"}]} + except requests.exceptions.Timeout as errt: + logging.error("A Timeout Error occurred:" + repr(errt)) + return {"errors": [ { "type": "Timeout Error"}]} + except requests.exceptions.RequestException as err: + logging.error("An Unknown Error occurred" + repr(err)) + return {"errors": [ { "type": "Request Exception"}]} + +""" function to add commas for prettier output""" +def add_comma(s): + if s != '': + s = ', ' + s + return s + else: + return s + +output = [] """ find all URLs of the form https://github.com/owner/repo """ -with open('README.md', 'r') as readme: - data = readme.read() - project_urls = re.findall('https://github.com/[A-z]*/[A-z|0-9|\-|_|\.]+', data) +def parse_github_projects(): + with open(sys.argv[1], 'r') as readme: + logging.info('Testing ' + sys.argv[1]) + data = readme.read() + #project_urls = re.findall('https://github\.com/.*', data) + project_urls = re.findall('https://github\.com/([a-zA-Z\d\-\._]{1,39}/[a-zA-Z\d\-\._]{1,39})(?=\)|/|#\s)', data) + logging.info('Checking ' + str(len(project_urls)) + ' github repos.') + return sorted(set(project_urls)) -urls = sorted(set(project_urls)) +urls = parse_github_projects() + +""" function to check remaining rate limit """ +def check_github_remaining_limit(): + query = ''' + query{ + viewer { + login + } + rateLimit { + cost + remaining + resetAt + } + }''' + logging.info("Checking github api remaining rate limit.") + result = query_github_api(query, '') + if 'errors' in result: + logging.error(result["errors"][0]["type"] + ", " + result["errors"][0]["message"]) + with open('github_commit_dates.md', 'w') as filehandle: + filehandle.write('%s\n' % '--------------------\n### Github commit date checks') + filehandle.write(result["errors"][0]["type"] + ", " + result["errors"][0]["message"]) + else: + if result["data"]["rateLimit"]["remaining"] < len(urls): + logging.error('Github api calls remaining is insufficient, exiting.') + logging.error('URLS: ' + str(len(urls)) + ', api calls remaining: ' + str(result["data"]["rateLimit"]["remaining"]) + ', Resets at: ' + str(result["data"]["rateLimit"]["resetAt"])) + with open('github_commit_dates.md', 'w') as filehandle: + filehandle.write('%s\n' % '--------------------\n### Github commit date checks') + filehandle.write('Github api calls remaining is insufficient, exiting.\n') + filehandle.write('URLS: ' + str(len(urls)) + ', api calls remaining: ' + str(result["data"]["rateLimit"]["remaining"]) + ', Resets at: ' + str(result["data"]["rateLimit"]["resetAt"]) + '\n') + sys.exit(1) """ Uncomment this to debug the list of matched URLs """ # print(str(urls)) +# print(len(urls)) +# with open('links.txt', 'w') as filehandle: +# for l in urls: +# filehandle.write('%s\n' % l) + # exit(0) -""" login to github API """ -g = Github(access_token) - +check_github_remaining_limit() +i = 0 """ load project metadata, output last commit date and URL """ for url in urls: - project = re.sub('https://github.com/', '', url) - repo = g.get_repo(project) - print(str(repo.pushed_at) + ' https://github.com/' + project) + split = url.split("/") + variables = { + "owner": split[0], + "name": split[1] + } + query = ''' + query($owner: String!, $name: String!){ + repository(owner:$owner, name:$name) { + pushedAt + updatedAt + isArchived + isDisabled + nameWithOwner + } + rateLimit { + cost + remaining + resetAt + } + }''' + + github_graphql_data = query_github_api(query, variables) + if 'errors' in github_graphql_data: + logging.info(github_graphql_data["errors"][0]["type"]) + output.append([date(1900, 1, 1),'https://github.com/'+url, github_graphql_data["errors"][0]["type"]]) + else: + has_issue = False + note = '' + if github_graphql_data["data"]["repository"]["isArchived"] == True: + has_issue = True + note = 'Archived' + if github_graphql_data["data"]["repository"]["isDisabled"] == True: + if note == '': + has_issue = True + note = 'Disabled' + else: + note = note + ', Disabled' + if github_graphql_data["data"]["repository"]["nameWithOwner"] != url: + if note == '': + has_issue = True + note = 'Moved to https://github.com/'+ github_graphql_data["data"]["repository"]["nameWithOwner"] + else: + note = note + ', Moved to https://github.com/'+ github_graphql_data["data"]["repository"]["nameWithOwner"] + project_pushed_at = datetime.strptime(github_graphql_data["data"]["repository"]["pushedAt"], '%Y-%m-%dT%H:%M:%SZ').date() + if project_pushed_at < (date.today() - timedelta(days = 365)): + has_issue = True + if has_issue: + output.append([project_pushed_at, 'https://github.com/'+url, note]) + logging.info(str(project_pushed_at)+' | https://github.com/'+url+' | '+note) + i += 1 + + +if i > 0: + sorted_list = sorted(output, key=lambda x: x[0]) + with open('github_commit_dates.md', 'w') as filehandle: + filehandle.write('%s\n' % '--------------------\n### Github commit date checks') + filehandle.write('%s\n' % '#### There were %s repos last updated over 1 year ago.' % str(i)) + for l in sorted_list: + filehandle.write('* [ ] %s, %s%s \n' % (str(l[0]), l[1], add_comma(l[2]))) + sys.exit(0) +else: + with open('github_commit_dates.md', 'w') as filehandle: + filehandle.write('%s\n' % '--------------------\n### Github commit date checks') + filehandle.write('%s\n' % '#### There were no repos last updated over 1 year ago.') + diff --git a/tests/link_whitelist.txt b/tests/link_whitelist.txt new file mode 100644 index 00000000..1b9f1730 --- /dev/null +++ b/tests/link_whitelist.txt @@ -0,0 +1 @@ +flaskbb.org,nitter.net,airsonic.github.io/docs/apps \ No newline at end of file diff --git a/tests/test.js b/tests/test.js index 36ee3e4e..d20e96ce 100644 --- a/tests/test.js +++ b/tests/test.js @@ -8,6 +8,7 @@ let licenses = new Set(); let pr = false; let readme; let diff; +let mdOutput = []; //Parse the command options and set the pr var function parseArgs(args) { @@ -42,10 +43,11 @@ function split(text) { // All entries should match this pattern. If matches pattern returns true. function findPattern(text) { - const patt = /^\s{0,2}-\s\[.*?\]\(.*?\) (`⚠` )?- .{0,249}?\.( \(\[(Demo|Source Code|Clients)\]\([^)\]]*\)(, \[(Source Code|Clients)\]\([^)\]]*\))?(, \[(Source Code|Clients)\]\([^)\]]*\))*\))? \`.*?\` \`.*?\`$/; + const patt = /^\s{0,2}-\s\[.*?\]\(.*?\) (`⚠` )?- .{0,249}?\.( \(\[(Demo|Source Code|Clients)\]\([^)\]]*\)(, \[(Source Code|Clients)\]\([^)\]]*\))?(, \[(Source Code|Clients)\]\([^)\]]*\))*\))? \`.*?\` \`.*?\`$/m; if (patt.test(text) === true) { return true; } + console.log("Failed: "+text) return false; } @@ -61,9 +63,9 @@ function testMainLink(text) { const testA1 = /(- \W?\w*\W{0,2}.*?\)?)( .*$)/; if (!testA.test(text)) { let a1 = testA1.exec(text)[2]; - return chalk.red(text.replace(a1, '')) + return [chalk.red(text.replace(a1, '')), '🢂' + text.replace(a1, '') + '🢀'] } - return chalk.green(testA.exec(text)[1]) + return [chalk.green(testA.exec(text)[1]), testA.exec(text)[1]] } //Test '`⚠` - Short description, less than 250 characters.' @@ -74,23 +76,23 @@ function testDescription(text) { if (!testB.test(text)) { let b1 = testA1.exec(text)[1]; let b2 = testB2.exec(text)[1]; - return chalk.red(text.replace(b1, '').replace(b2, '')) + return [chalk.red(text.replace(b1, '').replace(b2, '')), '🢂' + text.replace(b1, '').replace(b2, '') + '🢀' ] } - return chalk.green(testB.exec(text)[1]) + return [chalk.green(testB.exec(text)[1]), testB.exec(text)[1]] } //If present, tests '([Demo](http://url.to/demo), [Source Code](http://url.of/source/code), [Clients](https://url.to/list/of/related/clients-or-apps))' function testSrcDemCli(text) { let testC = text.search(/\.\ \(|\.\ \[|\ \(\[[sSdDcC]/); // /\(\[|\)\,|\)\)/); - let testD = /(?<=\w. )(\(\[(Demo|Source Code|Clients)\]\([^)\]]*\)(, \[(Source Code|Clients)\]\([^)\]]*\))?(, \[(Source Code|Clients)\]\([^)\]]*\))*\))(?= \`?)/; + let testD = /(?<=\w. )(\(\[(Demo|Source Code|Clients)\]\([^)\]]*\)(, \[(Source Code|Clients)\]\([^)\]]*\))?(, \[(Source Code|Clients)\]\([^)\]]*\))*\) )(?=\`?)/; const testD1 = /(^- \W[a-zA-Z0-9-_ .]*\W{0,2}http[^\[]*)(?<= )/; - const testD2 = /(\`.*\` \`.*\`$)/; + const testD2 = /\ ?(\`.*\` \`.*\`$)/; if ((testC > -1) && (!testD.test(text))) { let d1 = testD1.exec(text)[1]; let d2 = testD2.exec(text)[1]; - return chalk.red(text.replace(d1, '').replace(d2, '')) + return [chalk.red(text.replace(d1, '').replace(d2, '')), '🢂' + text.replace(d1, '').replace(d2, '') + '🢀'] } else if (testC > -1) { - return chalk.green(testD.exec(text)[1]) + return [chalk.green(testD.exec(text)[1]), testD.exec(text)[1]] } return "" } @@ -102,19 +104,18 @@ function testLangLic(text) { const testE1 = /(^[^`]*)/; if (!testE) { let e1 = testE1.exec(text)[1]; - return chalk.red(text.replace(e1, '')) + return [chalk.red(text.replace(e1, '')), '🢂' + text.replace(e1, '') + '🢀'] } - return chalk.green(testD2.exec(text)[1]) + return [chalk.green(testD2.exec(text)[1]), + testD2.exec(text)[1]] } //Runs all the syntax tests... function findError(text) { - let res - res = testMainLink(text) - res += testDescription(text) - res += testSrcDemCli(text) - res += testLangLic(text) - return res + `\n` + resMainLink = testMainLink(text) + resDesc= testDescription(text) + resSrcDemCli= testSrcDemCli(text) + resLangLic= testLangLic(text) + return [resMainLink[0] + resDesc[0] + resSrcDemCli[0] + resLangLic[0] + `\n`, '```' + resMainLink[1] + resDesc[1] + resSrcDemCli[1] + resLangLic[1] + '```'] } //Check if license is in the list of licenses. @@ -122,7 +123,7 @@ function testLicense(md) { let pass = true; let lFailed = [] let lPassed = [] - const regex = /.*\`(.*)\` .*$/; + const regex = /.*?\`([a-zA-Z0-9\-\./]*)\`.+$/; try { for (l of regex.exec(md)[1].split("/")) { if (!licenses.has(l)) { @@ -136,11 +137,6 @@ function testLicense(md) { console.log(chalk.yellow("Error in License syntax, license not checked against list.")) return [false, "", ""] } - - - - - return [pass, lFailed, lPassed] } @@ -195,14 +191,15 @@ function entryErrorCheck() { e.pass = true e.name = parseName(e.raw) if (!findPattern(e.raw)) { - e.highlight = findError(e.raw); + errorRes = findError(e.raw); + e.highlight = errorRes[0]; e.pass = false; console.log(e.highlight) } e.licenseTest = testLicense(e.raw); if (!e.licenseTest) { e.pass = false; - console.log(chalk.red(`${e.name}'s license is not on License list.`)) + console.log(chalk.red(`${e.name}'s license is not on the License list.`)) } if (e.pass) { totalPass++ @@ -210,6 +207,7 @@ function entryErrorCheck() { totalFail++ } } + } else { console.log(chalk.cyan("Testing entire README.md\n")) total = entries.length @@ -217,7 +215,9 @@ function entryErrorCheck() { e.pass = true e.name = parseName(e.raw) if (!findPattern(e.raw)) { - e.highlight = findError(e.raw); + errorRes = findError(e.raw); + e.highlight = errorRes[0]; + mdOutput.push("* [ ] Line: " + e.line + ": " + e.name + "\n" + errorRes[1]); e.pass = false; console.log(`${chalk.yellow(e.line + ": ")}${e.highlight}`); syntax = e.highlight; @@ -226,6 +226,7 @@ function entryErrorCheck() { if (!e.licenseTest[0]) { e.pass = false; console.log(chalk.yellow(e.line + ": ") + `${e.name}'s license ${chalk.red(`'${e.licenseTest[1]}'`)} is not on the License list.\n`) + mdOutput.push("* [ ] Line: " + e.line + "\n" + e.name + "'s license is not on the License list.") } if (e.pass) { totalPass++ @@ -238,6 +239,10 @@ function entryErrorCheck() { console.log(chalk.blue(`\n-----------------------------\n`)) console.log(chalk.red(`${totalFail} Failed, `) + chalk.green(`${totalPass} Passed, `) + chalk.blue(`of ${total}`)) console.log(chalk.blue(`\n-----------------------------\n`)) + fs.writeFileSync('syntax_check.md', `--------------------\n### Syntax Checks\n#### ${totalFail} Failed, ${totalPass} Passed, of ${total}.\n`) + mdOutput.forEach(element => { + fs.appendFileSync('syntax_check.md', `${element}\n`) + }); process.exit(1); } else { console.log(chalk.blue(`\n-----------------------------\n`)) From 27071cb9c8a16ae69029c94fcad3b238c2504c9b Mon Sep 17 00:00:00 2001 From: Nathan Rew Date: Wed, 30 Jun 2021 15:28:52 -0500 Subject: [PATCH 2/6] Add github actions workflows Signed-off-by: Nathan Rew --- .github/workflows/monthly.yml | 6 +- tests/check-github-commit-dates.py | 203 +++++++++++++++-------------- 2 files changed, 111 insertions(+), 98 deletions(-) diff --git a/.github/workflows/monthly.yml b/.github/workflows/monthly.yml index 7de13091..79b411bd 100644 --- a/.github/workflows/monthly.yml +++ b/.github/workflows/monthly.yml @@ -20,11 +20,13 @@ jobs: printf '%s\n%s%s %s\n%s\n%s\n' '---' 'title: Monthly Checks - ' $( date +"%B %Y" ) 'labels: automated issue' '---' > .github/ISSUE_TEMPLATE.md echo -e '[![Monthly Checks](https://github.com/n8225/awesome-selfhosted/actions/workflows/monthly.yml/badge.svg)](https://github.com/n8225/awesome-selfhosted/actions/workflows/monthly.yml)' >> .github/ISSUE_TEMPLATE.md echo -e '\n--------------------' >> .github/ISSUE_TEMPLATE.md - echo -e '\n### Awersome_Bot link checks\n' >> .github/ISSUE_TEMPLATE.md + echo -e '\n### Awesome_Bot link checks\n' >> .github/ISSUE_TEMPLATE.md jq -r '.[] | ["* [ ] ", "Line ", .loc, ": ", .link, ", ", .error] | join("")' ab-results-README.md-filtered.json >> .github/ISSUE_TEMPLATE.md || true + echo -e '\n' >> .github/ISSUE_TEMPLATE.md cat github_commit_dates.md >> .github/ISSUE_TEMPLATE.md || true + echo -e '\n' >> .github/ISSUE_TEMPLATE.md cat syntax_check.md >> .github/ISSUE_TEMPLATE.md || true - echo -e '\n--------------------' >> .github/ISSUE_TEMPLATE.md + echo -e '\n--------------------\n' >> .github/ISSUE_TEMPLATE.md printf '%s/%s%s%s' ${GITHUB_SERVER_URL} ${GITHUB_REPOSITORY} '/actions/runs/' ${GITHUB_RUN_ID} >> .github/ISSUE_TEMPLATE.md - name: Verify template run: cat .github/ISSUE_TEMPLATE.md diff --git a/tests/check-github-commit-dates.py b/tests/check-github-commit-dates.py index f88a2034..932975a6 100755 --- a/tests/check-github-commit-dates.py +++ b/tests/check-github-commit-dates.py @@ -18,7 +18,7 @@ Usage: """ -import json +import math import sys import re import os @@ -40,18 +40,36 @@ __status__ = "Production" ############################################################################### logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S') +""" find all URLs of the form https://github.com/owner/repo """ +def parse_github_projects(): + with open(sys.argv[1], 'r') as readme: + logging.info('Testing ' + sys.argv[1]) + data = readme.read() + project_urls = re.findall('https://github\.com/([a-zA-Z\d\-\._]{1,39}/[a-zA-Z\d\-\._]{1,39})(?=\)|/|#\s)', data) + """ Uncomment this to debug the list of matched URLs """ + # print(str(project_urls)) + # print(len(project_urls)) + # with open('links.txt', 'w') as filehandle: + # for l in project_urls: + # filehandle.write('%s\n' % l) + + # exit(0) + sorted_urls = sorted(set(project_urls)) + logging.info('Checking ' + str(len(sorted_urls)) + ' github repos.') + return sorted_urls + + """ function to query Github graphql API """ -def query_github_api(query, variables): +def query_github_api(query): access_token = os.environ['GITHUB_TOKEN'] headers = {"Authorization": "Bearer " + access_token} github_adapter = HTTPAdapter(max_retries=7) session = requests.Session() session.mount('https:api.github.com/graphql', github_adapter) try: - logging.info('Querying API for %s', variables) - response = session.post('https://api.github.com/graphql', timeout=(10) , json={'query': query, 'variables': variables}, headers=headers) + response = session.post('https://api.github.com/graphql', timeout=(10) , json={'query': query}, headers=headers) response.raise_for_status() - logging.debug(response.json()) + #logging.debug(response.json()) return response.json() except requests.exceptions.HTTPError as errh: logging.error("An Http Error occurred:" + repr(errh)) @@ -74,22 +92,8 @@ def add_comma(s): else: return s -output = [] - -""" find all URLs of the form https://github.com/owner/repo """ -def parse_github_projects(): - with open(sys.argv[1], 'r') as readme: - logging.info('Testing ' + sys.argv[1]) - data = readme.read() - #project_urls = re.findall('https://github\.com/.*', data) - project_urls = re.findall('https://github\.com/([a-zA-Z\d\-\._]{1,39}/[a-zA-Z\d\-\._]{1,39})(?=\)|/|#\s)', data) - logging.info('Checking ' + str(len(project_urls)) + ' github repos.') - return sorted(set(project_urls)) - -urls = parse_github_projects() - """ function to check remaining rate limit """ -def check_github_remaining_limit(): +def check_github_remaining_limit(urls, project_per_call): query = ''' query{ viewer { @@ -102,7 +106,7 @@ def check_github_remaining_limit(): } }''' logging.info("Checking github api remaining rate limit.") - result = query_github_api(query, '') + result = query_github_api(query) if 'errors' in result: logging.error(result["errors"][0]["type"] + ", " + result["errors"][0]["message"]) with open('github_commit_dates.md', 'w') as filehandle: @@ -115,84 +119,91 @@ def check_github_remaining_limit(): with open('github_commit_dates.md', 'w') as filehandle: filehandle.write('%s\n' % '--------------------\n### Github commit date checks') filehandle.write('Github api calls remaining is insufficient, exiting.\n') - filehandle.write('URLS: ' + str(len(urls)) + ', api calls remaining: ' + str(result["data"]["rateLimit"]["remaining"]) + ', Resets at: ' + str(result["data"]["rateLimit"]["resetAt"]) + '\n') + filehandle.write('URLS: ' + str(len(urls)) + str(math.ceil(len(urls) / project_per_call)) + ', Github API cost: ' + ', api calls remaining: ' + str(result["data"]["rateLimit"]["remaining"]) + ', Resets at: ' + str(result["data"]["rateLimit"]["resetAt"]) + '\n') sys.exit(1) -""" Uncomment this to debug the list of matched URLs """ -# print(str(urls)) -# print(len(urls)) -# with open('links.txt', 'w') as filehandle: -# for l in urls: -# filehandle.write('%s\n' % l) - -# exit(0) - -check_github_remaining_limit() -i = 0 -""" load project metadata, output last commit date and URL """ -for url in urls: - split = url.split("/") - variables = { - "owner": split[0], - "name": split[1] - } - query = ''' - query($owner: String!, $name: String!){ - repository(owner:$owner, name:$name) { - pushedAt - updatedAt - isArchived - isDisabled - nameWithOwner - } - rateLimit { - cost - remaining - resetAt - } - }''' - - github_graphql_data = query_github_api(query, variables) - if 'errors' in github_graphql_data: - logging.info(github_graphql_data["errors"][0]["type"]) - output.append([date(1900, 1, 1),'https://github.com/'+url, github_graphql_data["errors"][0]["type"]]) - else: - has_issue = False - note = '' - if github_graphql_data["data"]["repository"]["isArchived"] == True: - has_issue = True - note = 'Archived' - if github_graphql_data["data"]["repository"]["isDisabled"] == True: - if note == '': - has_issue = True - note = 'Disabled' +def parse_api_output(github_graphql_data, url_store): + output = [] + if "errors" in github_graphql_data: + for e in github_graphql_data["errors"]: + print(e) + logging.info('https://github.com/'+ url_store[e["path"][0]] + ", " + e["type"]) + output.append([date(1900, 1, 1),'https://github.com/'+ url_store[e["path"][0]], e["type"]]) + if "data" in github_graphql_data: + for g, v in github_graphql_data["data"].items(): + if github_graphql_data["data"][g] == None: + continue + elif g == 'rateLimit': + logging.info('Remaining Ratelimit: ' + str(github_graphql_data["data"][g]["remaining"]) + ' Cost: ' + str(github_graphql_data["data"][g]["cost"])) else: - note = note + ', Disabled' - if github_graphql_data["data"]["repository"]["nameWithOwner"] != url: - if note == '': - has_issue = True - note = 'Moved to https://github.com/'+ github_graphql_data["data"]["repository"]["nameWithOwner"] - else: - note = note + ', Moved to https://github.com/'+ github_graphql_data["data"]["repository"]["nameWithOwner"] - project_pushed_at = datetime.strptime(github_graphql_data["data"]["repository"]["pushedAt"], '%Y-%m-%dT%H:%M:%SZ').date() - if project_pushed_at < (date.today() - timedelta(days = 365)): - has_issue = True - if has_issue: - output.append([project_pushed_at, 'https://github.com/'+url, note]) - logging.info(str(project_pushed_at)+' | https://github.com/'+url+' | '+note) + has_issue = False + note = '' + if github_graphql_data["data"][g]["isArchived"] == True: + has_issue = True + note = 'Archived' + if github_graphql_data["data"][g]["isDisabled"] == True: + if note == '': + has_issue = True + note = 'Disabled' + else: + note = note + ', Disabled' + if github_graphql_data["data"][g]["nameWithOwner"] != url_store[g]: + if note == '': + has_issue = True + note = 'Moved to https://github.com/'+ github_graphql_data["data"][g]["nameWithOwner"] + else: + note = note + ', Moved to https://github.com/'+ github_graphql_data["data"][g]["nameWithOwner"] + project_pushed_at = datetime.strptime(github_graphql_data["data"][g]["pushedAt"], '%Y-%m-%dT%H:%M:%SZ').date() + if project_pushed_at < (date.today() - timedelta(days = 365)): + has_issue = True + if has_issue: + output.append([project_pushed_at, 'https://github.com/'+url_store[g], note]) + logging.info(str(project_pushed_at)+' | https://github.com/'+url_store[g]+' | '+note) + return output + +def github_api_alias(url): + replace = ["-", "/", "."] + for s in replace: + url = url.replace(s, "_") + return "_" + url + +def build_query(urls, project_per_call): + i = 0 + output = [] + query_param = '{pushedAt updatedAt isArchived isDisabled nameWithOwner}' + url_store = {} + while (i < len(urls)): + query_repo_count = 0 + query = "query{rateLimit{cost remaining resetAt}" + while (query_repo_count < project_per_call and i < len(urls)): + key = github_api_alias(urls[i]) + url_store[key] = urls[i] + split = urls[i].split("/") + query += key + ':' + 'repository(owner:"' + split[0] + '" name:"' + split[1] + '")' + query_param + query_repo_count += 1 i += 1 + query += "}" + output.extend(parse_api_output(query_github_api(query), url_store)) + logging.debug('Total: ' + str(len(urls)) + ' Checked: ' + str(len(url_store))) + return output +def main(): + project_per_call = 100 + urls = parse_github_projects() + check_github_remaining_limit(urls, project_per_call) + output = build_query(urls, project_per_call) + if len(output) > 0: + sorted_list = sorted(output, key=lambda x: x[0]) + with open('github_commit_dates.md', 'w') as filehandle: + filehandle.write('%s\n' % '--------------------\n### Github commit date checks') + filehandle.write('%s\n' % '#### There were %s repos with issues.' % str(len(output))) + for l in sorted_list: + filehandle.write('* [ ] %s, %s%s \n' % (str(l[0]), l[1], add_comma(l[2]))) + sys.exit(1) + else: + with open('github_commit_dates.md', 'w') as filehandle: + filehandle.write('%s\n' % '--------------------\n### Github commit date checks') + filehandle.write('%s\n' % '#### There were no repos with issues.') + exit(0) -if i > 0: - sorted_list = sorted(output, key=lambda x: x[0]) - with open('github_commit_dates.md', 'w') as filehandle: - filehandle.write('%s\n' % '--------------------\n### Github commit date checks') - filehandle.write('%s\n' % '#### There were %s repos last updated over 1 year ago.' % str(i)) - for l in sorted_list: - filehandle.write('* [ ] %s, %s%s \n' % (str(l[0]), l[1], add_comma(l[2]))) - sys.exit(0) -else: - with open('github_commit_dates.md', 'w') as filehandle: - filehandle.write('%s\n' % '--------------------\n### Github commit date checks') - filehandle.write('%s\n' % '#### There were no repos last updated over 1 year ago.') - +main() From becb838708dd3175fb3bedd3b62cb1db82a19523 Mon Sep 17 00:00:00 2001 From: Nathan Rew Date: Wed, 30 Jun 2021 16:40:03 -0500 Subject: [PATCH 3/6] fix pull_request workflow Signed-off-by: Nathan Rew --- .github/workflows/pull_request.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5152a8e7..5923add4 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -2,7 +2,7 @@ name: Pull Request Checks on: pull_request: - branches: [ main ] + branches: [ master ] jobs: check_syntax: @@ -13,12 +13,14 @@ jobs: uses: actions/setup-node@v1 with: node-version: '14.x' + - name: create diff + run: git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md - name: Checks run: | cd test npm install chalk cd .. - git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md + script -e -c 'node tests/test.js -r README.md -d temp.md' check_links: @@ -29,6 +31,8 @@ jobs: uses: actions/setup-ruby@v1 with: ruby-version: 2.6.x + - name: create diff + run: git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md - name: Checks run: | gem install awesome_bot From af89e7930fa66e1d0726c7548563a229b135a3e5 Mon Sep 17 00:00:00 2001 From: Nathan Rew Date: Wed, 30 Jun 2021 16:49:40 -0500 Subject: [PATCH 4/6] fix pull_request workflow Signed-off-by: Nathan Rew --- .github/workflows/pull_request.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5923add4..31e6d5b4 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -14,13 +14,14 @@ jobs: with: node-version: '14.x' - name: create diff - run: git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md + run: | + git fetch + git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md - name: Checks run: | - cd test + cd tests npm install chalk cd .. - script -e -c 'node tests/test.js -r README.md -d temp.md' check_links: @@ -32,7 +33,9 @@ jobs: with: ruby-version: 2.6.x - name: create diff - run: git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md + run: | + git fetch + git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md - name: Checks run: | gem install awesome_bot From 5debc514d6e213850dfedc042a7fd6ad2bf1786b Mon Sep 17 00:00:00 2001 From: Nathan Rew Date: Wed, 30 Jun 2021 19:48:26 -0500 Subject: [PATCH 5/6] fix pull_request workflow Signed-off-by: Nathan Rew --- .github/workflows/pull_request.yml | 34 +++++++++++++++++------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 31e6d5b4..52f848ce 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -3,41 +3,45 @@ name: Pull Request Checks on: pull_request: branches: [ master ] + workflow_dispatch: jobs: check_syntax: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: create diff + run: git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" > temp.md - name: Use Node.js uses: actions/setup-node@v1 with: node-version: '14.x' - - name: create diff - run: | - git fetch - git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md - - name: Checks + - name: install chalk run: | cd tests npm install chalk cd .. - script -e -c 'node tests/test.js -r README.md -d temp.md' + - name: Checks + run: script -e -c 'node tests/test.js -r README.md -d temp.md' check_links: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: create diff + run: git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" > temp.md - name: Set up Ruby 2.6 uses: actions/setup-ruby@v1 with: ruby-version: 2.6.x - - name: create diff - run: | - git fetch - git diff origin/master -U0 README.md | grep --perl-regexp --only-matching "(?<=^\+).*" >> temp.md + - name: install awesome_bot + run: gem install awesome_bot - name: Checks - run: | - gem install awesome_bot - awesome_bot -f temp.md --allow-redirect --skip-save-results --allow 202 --white-list < tests/awesomebot_whitelist.txt + run: awesome_bot -f temp.md --allow-redirect --skip-save-results --allow 202 --white-list < tests/link_whitelist.txt \ No newline at end of file From 329d16b33040d7e7b9a4cdc7e46d248897ca58e2 Mon Sep 17 00:00:00 2001 From: n8225 Date: Thu, 1 Jul 2021 07:22:27 -0500 Subject: [PATCH 6/6] Fix CRON --- .github/workflows/monthly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/monthly.yml b/.github/workflows/monthly.yml index 79b411bd..164bda46 100644 --- a/.github/workflows/monthly.yml +++ b/.github/workflows/monthly.yml @@ -2,7 +2,7 @@ name: Monthly Checks on: schedule: - - cron: '* * 1 * *' + - cron: '0 5 1 * *' workflow_dispatch: jobs: