tooling: add black and flake8 for python formatting/linting (#6454)

2021-09-03 15:17:51 +00:00 · 2021-09-03 15:17:51 +00:00 · 3fa29ea1c2
parent f4390b1637
commit 3fa29ea1c2
7 changed files with 188 additions and 123 deletions
--- a/.flake8
+++ b/.flake8
@ -0,0 +1,5 @@
+[flake8]
+max-line-length = 88
+# We ignore E501 as black handles it for us, and in a way that ignores strings
+# that go over the line length, as opposed to flake8 which flags such strings.
+extend-ignore = E203,E501
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -9,6 +9,10 @@ jobs:
    name: CI

    steps:
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8

    - name: Checkout
      uses: actions/checkout@v2
@ -22,6 +26,9 @@ jobs:
    - name: Install npm dependencies
      run: npm ci

+    - name: Install python dependencies
+      run: python3 -m pip install -r requirements.txt
+
    - name: Test
      run: npm test

--- a/requirements.txt
+++ b/requirements.txt
@ -0,0 +1,2 @@
+black==21.8b0
+flake8==3.9.2
--- a/scripts/pdf/render.py
+++ b/scripts/pdf/render.py
@ -15,7 +15,8 @@ import markdown
 import argparse
 from datetime import datetime

-from weasyprint import HTML, CSS
+from weasyprint import HTML
+

 def main(loc, colorscheme):

@ -30,32 +31,42 @@ def main(loc, colorscheme):
        csslist.append(colorscheme + ".css")

    # A string that stores all pages in HTML format
-    html = '<!doctype html><html><head><meta charset="utf-8"></head>' \
-        +"<body><h1 class=title-main>tldr pages</h1>" \
-        + "<h4 class=title-sub>Simplified and community-driven man pages</h4>" \
-        + "<h6 class=title-sub><em><small>Generated on " + datetime.now().strftime("%c") + "</small></em></h6>" \
+    html = (
+        '<!doctype html><html><head><meta charset="utf-8"></head>'
+        + "<body><h1 class=title-main>tldr pages</h1>"
+        + "<h4 class=title-sub>Simplified and community-driven man pages</h4>"
+        + "<h6 class=title-sub><em><small>Generated on "
+        + datetime.now().strftime("%c")
+        + "</small></em></h6>"
        + '<p style="page-break-before: always" ></p>'
+    )

    # Writing names of all directories inside 'pages' to a list
    for operating_sys in sorted(os.listdir(loc)):
-        
+
        # Required string to create directory title pages
-        html += "<h2 class=title-dir>" + operating_sys.capitalize() + "</h2>" \
+        html += (
+            "<h2 class=title-dir>"
+            + operating_sys.capitalize()
+            + "</h2>"
            + '<p style="page-break-before: always" ></p>'
+        )

        # Conversion of Markdown to HTML string
-        for page_number, md in enumerate(sorted(glob.glob(os.path.join(loc, operating_sys, "*.md"))), start=1):
+        for page_number, md in enumerate(
+            sorted(glob.glob(os.path.join(loc, operating_sys, "*.md"))), start=1
+        ):
            with open(md, "r") as inp:
                text = inp.readlines()
                for line in text:
-                    if re.match(r'^>', line):
-                        line = line[:0] + '####' + line[1:]
+                    if re.match(r"^>", line):
+                        line = line[:0] + "####" + line[1:]
                    html += markdown.markdown(line)
            html += '<p style="page-break-before: always" ></p>'
            print(f"Rendered page {page_number} of the directory {operating_sys}")
-    
+
    html += "</body></html>"
-    
+
    # Writing the PDF to disk
    print("\nConverting all pages to PDF...")
    HTML(string=html).write_pdf("tldr-pages.pdf", stylesheets=csslist)
@ -63,12 +74,22 @@ def main(loc, colorscheme):
    if os.path.exists("tldr-pages.pdf"):
        print("\nCreated tldr-pages.pdf in the current directory!\n")

+
 if __name__ == "__main__":

    # Parsing the arguments
-    parser = argparse.ArgumentParser(prog="tldr-pages-to-pdf", description="A Python script to generate a single PDF document with all the `tldr` pages.")
-    parser.add_argument("dir_path", help = "Path to the 'pages' directory")
-    parser.add_argument("-c", "--color", choices=["solarized-light", "solarized-dark", "basic"], default="basic", help="Color scheme of the PDF")
+    parser = argparse.ArgumentParser(
+        prog="tldr-pages-to-pdf",
+        description="A Python script to generate a single PDF document with all the `tldr` pages.",
+    )
+    parser.add_argument("dir_path", help="Path to the 'pages' directory")
+    parser.add_argument(
+        "-c",
+        "--color",
+        choices=["solarized-light", "solarized-dark", "basic"],
+        default="basic",
+        help="Color scheme of the PDF",
+    )
    args = parser.parse_args()

    main(args.dir_path, args.color)
--- a/scripts/send-to-bot.py
+++ b/scripts/send-to-bot.py
@ -5,9 +5,9 @@ import os
 import sys
 import requests

-BOT_URL = 'https://tldr-bot.starbeamrainbowlabs.com'
+BOT_URL = "https://tldr-bot.starbeamrainbowlabs.com"

-COMMENT_ERROR="""
+COMMENT_ERROR = """
 The [build](https://github.com/tldr-pages/tldr/actions/runs/{build_id}) for this PR failed with the following error(s):

 ```
@ -17,7 +17,7 @@ The [build](https://github.com/tldr-pages/tldr/actions/runs/{build_id}) for this
 Please fix the error(s) and push again.
 """

-COMMENT_CHECK="""
+COMMENT_CHECK = """
 Hello! I've noticed something unusual when checking this PR:

 {content}
@ -27,62 +27,69 @@ Is this intended? If so, just ignore this comment. Otherwise, please double-chec

 ################################################################################

+
 def post_comment(pr_id, body, once):
-  endpoint = BOT_URL + '/comment'
+    endpoint = BOT_URL + "/comment"

-  if once:
-    endpoint += '/once'
+    if once:
+        endpoint += "/once"

-  data = {'pr_id': pr_id, 'body': body}
+    data = {"pr_id": pr_id, "body": body}

-  try:
-    with requests.post(endpoint, json=data) as r:
-      if r.status_code != requests.codes.ok:
-        print('Error: tldr-bot responded with code', r.status_code, file=sys.stderr)
-        print(r.text, file=sys.stderr)
+    try:
+        with requests.post(endpoint, json=data) as r:
+            if r.status_code != requests.codes.ok:
+                print(
+                    "Error: tldr-bot responded with code",
+                    r.status_code,
+                    file=sys.stderr,
+                )
+                print(r.text, file=sys.stderr)
+                return False
+    except requests.exceptions.RequestException as e:
+        print("Error sending data to tldr-bot:", str(e), file=sys.stderr)
        return False
-  except requests.exceptions.RequestException as e:
-    print('Error sending data to tldr-bot:', str(e), file=sys.stderr)
-    return False
-    
-  return True
+
+    return True
+

 def main(action):
-  if action not in ('report-errors', 'report-check-results'):
-    print('Unknown action:', action, file=sys.stderr)
-    sys.exit(1)
+    if action not in ("report-errors", "report-check-results"):
+        print("Unknown action:", action, file=sys.stderr)
+        sys.exit(1)

-  content = sys.stdin.read().strip()
+    content = sys.stdin.read().strip()

-  if action == 'report-errors':
-    comment_body = COMMENT_ERROR.format(build_id=BUILD_ID, content=content)
-    comment_once = False
-  elif action == 'report-check-results':
-    comment_body = COMMENT_CHECK.format(content=content)
-    comment_once = True
+    if action == "report-errors":
+        comment_body = COMMENT_ERROR.format(build_id=BUILD_ID, content=content)
+        comment_once = False
+    elif action == "report-check-results":
+        comment_body = COMMENT_CHECK.format(content=content)
+        comment_once = True
+
+    if post_comment(PR_ID, comment_body, comment_once):
+        print("Success.")
+    else:
+        print("Error sending data to tldr-bot!", file=sys.stderr)

-  if post_comment(PR_ID, comment_body, comment_once):
-    print('Success.')
-  else:
-    print('Error sending data to tldr-bot!', file=sys.stderr)

 ################################################################################

-if __name__ == '__main__':
-  REPO_SLUG = os.environ.get('GITHUB_REPOSITORY')
-  PR_ID = os.environ.get('PULL_REQUEST_ID')
-  BUILD_ID = os.environ.get('GITHUB_RUN_ID')
+if __name__ == "__main__":
+    REPO_SLUG = os.environ.get("GITHUB_REPOSITORY")
+    PR_ID = os.environ.get("PULL_REQUEST_ID")
+    BUILD_ID = os.environ.get("GITHUB_RUN_ID")

-  if PR_ID is None or BUILD_ID is None or REPO_SLUG is None:
-    print('Needed environment variables are not set.', file=sys.stderr)
-    sys.exit(1)
+    if PR_ID is None or BUILD_ID is None or REPO_SLUG is None:
+        print("Needed environment variables are not set.", file=sys.stderr)
+        sys.exit(1)

-  if PR_ID is None or PR_ID == 'false':
-    print('Not a pull request, refusing to run.', file=sys.stderr)
-    sys.exit(0)
+    if PR_ID is None or PR_ID == "false":
+        print("Not a pull request, refusing to run.", file=sys.stderr)
+        sys.exit(0)

-  if len(sys.argv) != 2:
-    print('Usage:', sys.argv[0], '<ACTION>', file=sys.stderr)
-    sys.exit(1)
+    if len(sys.argv) != 2:
+        print("Usage:", sys.argv[0], "<ACTION>", file=sys.stderr)
+        sys.exit(1)

-  main(sys.argv[1])
+    main(sys.argv[1])
--- a/scripts/set-more-info-link.py
+++ b/scripts/set-more-info-link.py
@ -8,50 +8,49 @@ import subprocess
 import sys

 labels = {
-    'en': 'More information:',
-    'bs': 'Više informacija:',
-    'da': 'Mere information:',
-    'de': 'Weitere Informationen:',
-    'es': 'Más información:',
-    'fa': 'اطلاعات بیشتر:',
-    'fr': 'Plus d\'informations\xa0:',
-    'sh': 'Više informacija:',
-    'hi': 'अधिक जानकारी:',
-    'id': 'Informasi lebih lanjut:',
-    'it': 'Maggiori informazioni:',
-    'ja': '詳しくはこちら:',
-    'ko': '더 많은 정보:',
-    'ml': 'കൂടുതൽ വിവരങ്ങൾ:',
-    'nl': 'Meer informatie:',
-    'no': 'Mer informasjon:',
-    'pl': 'Więcej informacji:',
-    'pt_BR': 'Mais informações:',
-    'pt_PT': 'Mais informações:',
-    'ru': 'Больше информации:',
-    'sv': 'Mer information:',
-    'ta': 'மேலும் தகவல்:',
-    'th': 'ดูเพิ่มเติม:',
-    'tr': 'Daha fazla bilgi için:',
-    'zh_TW': '更多資訊：',
-    'zh': '更多信息：',
+    "en": "More information:",
+    "bs": "Više informacija:",
+    "da": "Mere information:",
+    "de": "Weitere Informationen:",
+    "es": "Más información:",
+    "fa": "اطلاعات بیشتر:",
+    "fr": "Plus d'informations\xa0:",
+    "sh": "Više informacija:",
+    "hi": "अधिक जानकारी:",
+    "id": "Informasi lebih lanjut:",
+    "it": "Maggiori informazioni:",
+    "ja": "詳しくはこちら:",
+    "ko": "더 많은 정보:",
+    "ml": "കൂടുതൽ വിവരങ്ങൾ:",
+    "nl": "Meer informatie:",
+    "no": "Mer informasjon:",
+    "pl": "Więcej informacji:",
+    "pt_BR": "Mais informações:",
+    "pt_PT": "Mais informações:",
+    "ru": "Больше информации:",
+    "sv": "Mer information:",
+    "ta": "மேலும் தகவல்:",
+    "th": "ดูเพิ่มเติม:",
+    "tr": "Daha fazla bilgi için:",
+    "zh_TW": "更多資訊：",
+    "zh": "更多信息：",
 }

-IGNORE_FILES = (
-    '.DS_Store',
-)
+IGNORE_FILES = (".DS_Store",)


 def get_tldr_root():
    # if this script is running from tldr/scripts, the parent's parent is the root
    f = os.path.normpath(__file__)
-    if f.endswith('tldr/scripts/set-more-info-link.py'):
+    if f.endswith("tldr/scripts/set-more-info-link.py"):
        return os.path.dirname(os.path.dirname(f))

-    if 'TLDR_ROOT' in os.environ:
-        return os.environ['TLDR_ROOT']
+    if "TLDR_ROOT" in os.environ:
+        return os.environ["TLDR_ROOT"]
    else:
        print(
-            '\x1b[31mPlease set TLDR_ROOT to the location of a clone of https://github.com/tldr-pages/tldr.')
+            "\x1b[31mPlease set TLDR_ROOT to the location of a clone of https://github.com/tldr-pages/tldr."
+        )
        sys.exit(1)


@ -64,39 +63,39 @@ def set_link(file, link):

    # find start and end of description
    for i, line in enumerate(lines):
-        if line.startswith('>') and desc_start == 0:
+        if line.startswith(">") and desc_start == 0:
            desc_start = i
-        if not lines[i + 1].startswith('>') and desc_start != 0:
+        if not lines[i + 1].startswith(">") and desc_start != 0:
            desc_end = i
            break

    # compute locale
    pages_dir = os.path.basename(os.path.dirname(os.path.dirname(file)))
-    if '.' in pages_dir:
-        _, locale = pages_dir.split('.')
+    if "." in pages_dir:
+        _, locale = pages_dir.split(".")
    else:
-        locale = 'en'
+        locale = "en"

    # build new line
    if locale == "zh" or locale == "zh_TW":
-        new_line = f'> {labels[locale]}<{link}>.\n'
+        new_line = f"> {labels[locale]}<{link}>.\n"
    else:
-        new_line = f'> {labels[locale]} <{link}>.\n'
+        new_line = f"> {labels[locale]} <{link}>.\n"

    if lines[desc_end] == new_line:
        # return empty status to indicate that no changes were made
-        return ''
+        return ""

-    if re.search(r'^>.*<.+>', lines[desc_end]):
+    if re.search(r"^>.*<.+>", lines[desc_end]):
        # overwrite last line
        lines[desc_end] = new_line
-        status = '\x1b[34mlink updated'
+        status = "\x1b[34mlink updated"
    else:
        # add new line
        lines.insert(desc_end + 1, new_line)
-        status = '\x1b[36mlink added'
+        status = "\x1b[36mlink added"

-    with open(file, 'w') as f:
+    with open(file, "w") as f:
        f.writelines(lines)

    return status
@ -104,32 +103,41 @@ def set_link(file, link):

 def main():
    parser = argparse.ArgumentParser(
-        description='Sets the "More information" link for all translations of a page')
-    parser.add_argument('-p', '--page', type=str, required=True,
-                        help='page name in the format "platform/command.md"')
-    parser.add_argument('-s', '--stage', action='store_true', default=False,
-                        help='stage modified pages (requires `git` to be on $PATH and TLDR_ROOT to be a Git repository)')
-    parser.add_argument('link', type=str)
+        description='Sets the "More information" link for all translations of a page'
+    )
+    parser.add_argument(
+        "-p",
+        "--page",
+        type=str,
+        required=True,
+        help='page name in the format "platform/command.md"',
+    )
+    parser.add_argument(
+        "-s",
+        "--stage",
+        action="store_true",
+        default=False,
+        help="stage modified pages (requires `git` to be on $PATH and TLDR_ROOT to be a Git repository)",
+    )
+    parser.add_argument("link", type=str)
    args = parser.parse_args()

    root = get_tldr_root()
-    pages_dirs = [d for d in os.listdir(root) if d.startswith('pages')]
+    pages_dirs = [d for d in os.listdir(root) if d.startswith("pages")]

    target_paths = []
    rel_paths = []

-    if not args.page.lower().endswith('.md'):
-        args.page = f'{args.page}.md'
+    if not args.page.lower().endswith(".md"):
+        args.page = f"{args.page}.md"

    for pages_dir in pages_dirs:
        pages_dir_path = os.path.join(root, pages_dir)
-        platforms = [i for i in os.listdir(
-            pages_dir_path) if i not in IGNORE_FILES]
+        platforms = [i for i in os.listdir(pages_dir_path) if i not in IGNORE_FILES]
        for platform in platforms:
            platform_path = os.path.join(pages_dir_path, platform)
            pages = os.listdir(platform_path)
-            commands = [
-                f'{platform}/{p}' for p in pages if p not in IGNORE_FILES]
+            commands = [f"{platform}/{p}" for p in pages if p not in IGNORE_FILES]
            if args.page in commands:
                path = os.path.join(pages_dir_path, args.page)
                target_paths.append(path)
@ -137,15 +145,15 @@ def main():
    target_paths.sort()

    for path in target_paths:
-        rel_path = path.replace(f'{root}/', '')
+        rel_path = path.replace(f"{root}/", "")
        rel_paths.append(rel_path)
        status = set_link(path, args.link)
-        if status != '':
-            print(f'\x1b[32m{rel_path} {status}\x1b[0m')
+        if status != "":
+            print(f"\x1b[32m{rel_path} {status}\x1b[0m")

    if args.stage:
-        subprocess.call(['git', 'add', *rel_paths], cwd=root)
+        subprocess.call(["git", "add", *rel_paths], cwd=root)


-if __name__ == '__main__':
+if __name__ == "__main__":
    main()
--- a/scripts/test.sh
+++ b/scripts/test.sh
@ -16,6 +16,21 @@ function run_tests {
  for f in ./pages.*; do
    tldr-lint --ignore "TLDR003,TLDR004,TLDR005,TLDR015,TLDR104" ${f}
  done
+  run_black
+  flake8 scripts
+}
+
+# Wrapper around black as it outputs everything to stderr,
+# but we want to only print if there are actual errors, and not
+# the "All done!" success message.
+function run_black {
+  # we want to ignore the exit code from black on failure, so that we can
+  # do the conditional printing below
+  errs=$(black scripts --check 2>&1 || true)
+  if [[ ${errs} != "All done!"* ]]; then
+     echo -e "${errs}" >&2
+     return 1
+  fi
 }

 # Special test function for GitHub Actions pull request builds.