Reimport files

.bin/check-file-for-starting-slash

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-# Script to verify if a file contain any entries starting with a slash and show a warning into the github action result UI.
-# Received as input a list of file paths to check separated by a space.
-# More precisely the result of the "tj-actions/changed-files" github action.
-## References:
-# See https://github.com/tj-actions/changed-files
-modified_files="$1"
-for modified_file in $modified_files
-do
-  echo "[+] Check $modified_file ..."
-  matches=$(grep -cE '^/[a-zA-Z0-9\._]+' $modified_file)
-  echo "Entries identified starting with a slash: $matches"
-  if [ $matches -ne 0 ]
-  then
-    echo "::warning file=$modified_file,line=1,col=1,endColumn=1::$matches entries start with a slash."
-  fi
-done

.bin/checkers/README.md

@@ -0,0 +1,45 @@
+All the files here will be run by `validators.sh` as part of the check pipeline
+
+## Specifications
+Scripts will be passed a space separated list of files to check
+e.g `./script.sh "Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt"`
+
+If you want your output to be parsed by the caller script, follow the below specs,else the output will be displayed raw in actions summaries
+
+## Wrapped calls
+
+### Wrapped calls checks
+
+- - -
+
+Scripts should check if its being run by the caller script. The environment variable `IS_RUNNING_UNDER_CALLER_SCRIPT` will be set to one
+
+### Wrapped calls Specifications
+
+- - -
+
+Checker scripts will now have to print out the name of the check followed by a new line
+
+Example `New line checker`
+
+This value will be used as the checker title
+
+After that, the descriptions will need to be printed out
+
+Example `To fix the error, you have to remove the empty lines or new lines at the end of the file`
+
+This is for contributors to understand why the checks failed
+
+- - -
+
+Outputs from now on will have to be in separate lines for each warnings or errors
+
+This is the format for errors
+
+Example `E,filename,line_number`
+
+In the above example, `E` stands for errors. Accepted values are `E` and `W` for errors and warnings respectively
+
+filename is the name of the file that the error comes from.
+
+line_number is the line the script detected the error in.

.bin/checkers/check-file-for-starting-slash.py

@@ -0,0 +1,66 @@
+#!/usr/bin/python3
+
+import os,sys
+
+if not sys.argv[1]:
+    exit(0)
+
+IS_WRAPPED=False
+
+if "IS_RUNNING_UNDER_CALLER_SCRIPT" in os.environ:
+    IS_WRAPPED=os.environ['IS_RUNNING_UNDER_CALLER_SCRIPT']=="1"
+
+def print_normal(msg):
+
+    if IS_WRAPPED:
+        return
+    print(msg)    
+
+def print_err(file,line_number):
+   
+    if IS_WRAPPED:
+        print("E,%s,%s"%(file,line_number))
+
+def print_warn(file,line_number):
+   
+    if IS_WRAPPED:
+        print("W,%s,%s"%(file,line_number))    
+
+print_normal("[+] Entries starting with slash checker")
+if IS_WRAPPED:
+    print("Entries starting with slash check")
+    print("The warning are more for informative purposes and does not actually serve as a check. You can ignore this.")
+
+files=sys.argv[1].split(" ")
+
+for i in files:
+    if not os.path.isfile(i):
+        print_err(i,0)
+        print_normal("[!] %s does not exist!"%(i))
+        exit(2)
+
+pass_status=True
+
+for i in files:
+    contents=open(i,"rb").read()
+    counter=1
+
+    for line in contents.splitlines():
+        if line.startswith(b'/'):
+            print_normal("[!] Warning: %s starts with a slash on line %i!"%(i,counter))
+            print_warn(i,counter)
+            pass_status=False
+
+        counter+=1
+    print_normal("[+] %s passed no starting slash check!"%(i))
+
+if pass_status:
+    print_normal("[+] All files passed checks")
+    exit(0)
+
+print_normal("[!] Warning: One or more files failed to pass the checks")
+
+if IS_WRAPPED:
+    exit(0)
+else:
+    exit(2)

.bin/checkers/new-line-and-empty-line-checker.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+import os,sys
+
+if not sys.argv[1]:
+    exit(0)
+
+IS_WRAPPED=False
+
+if "IS_RUNNING_UNDER_CALLER_SCRIPT" in os.environ:
+    IS_WRAPPED=os.environ['IS_RUNNING_UNDER_CALLER_SCRIPT']=="1"
+
+def print_normal(msg):
+
+    if IS_WRAPPED:
+        return
+    print(msg)    
+
+def print_err(file,line_number):
+   
+    if IS_WRAPPED:
+        print("E,%s,%s"%(file,line_number))
+
+def print_warn(file,line_number):
+   
+    if IS_WRAPPED:
+        print("W,%s,%s"%(file,line_number))    
+
+print_normal("[+] New line and empty line check")
+if IS_WRAPPED:
+    print("New line and empty line check")
+    print("To fix the error, you would have to remove the empty lines or new lines at the end of the file.")
+
+files=sys.argv[1].split(" ")
+
+for i in files:
+    if not os.path.isfile(i):
+        print_err(i,0)
+        print_normal("[!] %s does not exist!"%(i))
+        exit(2)
+
+overall_pass_status=True
+
+for i in files:
+    
+    contents=open(i,"rb").read()
+    line_number=len(contents.split(b'\n'))+1
+
+    if contents[-1:] == b'\n':
+        print_normal("[!] Warning: %s ends with a new line!"%(i))
+        print_warn(i,line_number)
+        overall_pass_status=False
+    else:
+        print_normal("[+] %s passed new line check!"%(i))
+
+    counter=1
+
+    line_pass_status=True
+
+    for line in contents.splitlines():
+        if not line:
+            print_normal("[!] Warning: %s has an empty entry on line %i!"%(i,counter))
+            print_warn(i,counter)
+            pass_status=False
+            line_pass_status=False
+            continue
+
+        elif not line.strip():
+            print_normal("[!] Warning: %s has an whitespace only entry on line %i!"%(i,counter))
+            print_warn(i,counter)
+            pass_status=False
+            line_pass_status=False
+            continue
+
+        counter+=1
+    if line_pass_status:
+        print_normal("[+] %s passed empty line check!"%(i))
+
+if overall_pass_status:
+    print_normal("[+] All files passed checks")
+    exit(0)
+
+print_normal("[!] Warning: One or more files failed to pass the checks")
+
+if IS_WRAPPED:
+    exit(0)
+else:
+    exit(2)

.bin/new-line-and-empty-line-checker.py

@@ -1,38 +0,0 @@
-#!/usr/bin/env python3
-
-# test string: ./.bin/new-line-checker.py "Fuzzing/file-extensions-all-cases.txt Fuzzing/file-extensions-lower-case.txt Fuzzing/file-extensions-upper-case.txt Fuzzing/file-extensions.txt"
-
-import os
-import sys
-
-print("[+] New line check")
-
-if not sys.argv[1]:
-    exit(0)
-
-files=sys.argv[1].split(" ")
-
-for i in files:
-    if not os.path.isfile(i):
-        print("[!] %s does not exist!"%(i))
-        exit(2)
-
-for i in files:
-    contents=open(i,"rb").read()
-
-    if contents[-1] == b'\n':
-        print("[!] %s ends with a new line!"%(i))
-        exit(2)
-    print("[+] %s passed new line check!"%(i))
-
-    counter=1
-
-    for line in contents.splitlines(False):
-        if len(line)==0:
-            print("[!] %s has an empty entry at line %i!"%(i,counter))
-            exit(2)
-        counter+=1
-    print("[+] %s passed empty line check!"%(i))
-
-print("[+] All files passed checks")
-# exit(0)

.bin/validators.py

@@ -0,0 +1,323 @@
+#!/usr/bin/python3
+
+# The workflow commands are from https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions
+# TL;DR You can echo text out and the text can trigger workflow commands if in format "::workflow-command parameter1={data},parameter2={data}::{command value}"
+# We are using the warnings and errors settings.
+# You may use the below lines for testing
+#
+# Invalid test args ./.bin/validators.py "Fuzzing/file-extensions-all-cases.txt Fuzzing/file-extensions-lower-case.txt Fuzzing/file-extensions-upper-case.txt Fuzzing/file-extensions.txt"
+# valid test args: ./.bin/validators.py "Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt"
+#
+# Script by @molangning. I am the guy you want to ping when this script get borked, not the repository owners
+
+import os,subprocess,sys
+
+args=sys.argv[1]
+files=[]
+STEP_SUMMARY_LOCATION="summary.md"
+IS_RUNNING_AS_ACTIONS=False
+CHECK_MARK_EMOJI=":white_check_mark:"
+CROSS_MARK_EMOJI=":negative_squared_cross_mark:"
+QUESTION_MARK_EMOJI=":question:"
+FORMATTED_OUTPUT_FORMAT="""
+### %s
+
+%s
+
+- - -
+
+Errors
+
+```
+%s
+```
+
+Warnings
+
+```
+%s
+```
+"""
+UNFORMATTED_OUTPUT_FORMAT="""
+### %s
+
+- - -
+
+Output
+
+```
+%s
+```
+"""
+SUMMARY_FORMAT="""
+# Checks summary
+
+This is a summary of the checks ran on the pushed files. If there is any errors please fix them before pushing again.
+## Check statuses
+
+| Test name | Passed? |
+| --------- | :-----: |
+%s
+
+## Errors and warnings
+%s
+
+## Other unformatted outputs
+This section is for scripts that doesn't follow specifications
+
+%s
+"""
+TABLE_FORMAT="| %s | %s |\n"
+WARN_MSG="Warnings in file %s on lines %s"
+ERROR_MSG="Errors in file %s on lines %s"
+WARNING_STRING="::warning file=%s,line=%s,col=%s,endColumn=%s::%s"
+ERROR_STRING="::error file=%s,line=%s,col=%s,endColumn=%s::%s"
+
+if "GITHUB_STEP_SUMMARY" not in os.environ:
+    print("[!] GITHUB_STEP_SUMMARY not found in system environments!")
+    print("[-] This error may occur if you are running this script in your own machine\n")
+else:
+    STEP_SUMMARY_LOCATION=os.environ["GITHUB_STEP_SUMMARY"]
+    IS_RUNNING_AS_ACTIONS=True
+
+if IS_RUNNING_AS_ACTIONS:
+    print("[+] Now running as github actions\n")
+else:
+    print("[+] Now running as normal user\n")
+
+def print_warn(file,msg,line=1,col=1,endcol=1):
+    if IS_RUNNING_AS_ACTIONS:
+        print(WARNING_STRING%(file,line,col,endcol,msg))
+    else:
+        print(msg)
+
+
+def print_err(file,msg,line=1,col=1,endcol=1):
+    if IS_RUNNING_AS_ACTIONS:
+        print(ERROR_STRING%(file,line,col,endcol,msg))
+    else:
+        print(msg)
+        
+for root,_,file_list in os.walk('.bin/checkers'):
+    for file in file_list:
+        if file.endswith('.md'):
+            continue
+
+        files.append(os.path.join(root,file))
+
+sys_env=os.environ.copy()
+sys_env.update({"IS_RUNNING_UNDER_CALLER_SCRIPT":"1"})
+all_events=[]
+
+for i in files:
+    events={"error":[],"warn":[],"raw_output":""}
+    is_using_wrapped_syntax=True
+    exec_status=1 # 0 for fail, 1 for success, 2 for unknown
+    description=""
+    
+    try:
+        output=subprocess.check_output([i,args],env=sys_env)
+
+    except PermissionError:
+        print_warn(i,"[!] Not running test %s due to insufficient privileges!"%(i))
+        print_warn(i,"[!] chmod +x it to let the workflow run the check.\n")
+        continue
+
+    except subprocess.CalledProcessError as exec_err:
+        print_err(i,"[!] Error! Calling command with args %s failed!"%(str([i,args])))
+        print_err(i,"[!] Process exited with error code %s and error message %s\n"%(exec_err.returncode,exec_err.output))
+        continue
+
+    output=output.decode('utf-8')
+    events["raw_output"]=output
+
+    split_output=output.split('\n')
+    
+    if split_output[-1]=="":
+        split_output=split_output[:-1]
+
+    if len(split_output)<2:
+        print_warn(i,"[!] Checker printed out less than two lines! Assuming not wrapped calls compliant.")
+        exec_status=2
+        is_using_wrapped_syntax=False
+
+    else:
+
+        for line in split_output[2:]:
+            
+            if not len(line):
+                continue
+
+            try:
+                event_type,file,line_number=line.split(',')
+            except:
+                print_warn(i,"[!] Split fail! Assuming checker %s is not wrapped calls compliant."%(i))
+                is_using_wrapped_syntax=False
+                exec_status=2
+                break
+
+            if event_type=="W":
+                events["warn"].append([file,line_number])
+                print_err(file,"[!] Checker %s got a warning for %s on line %s"%(i,file,line_number),line=line_number)
+            elif event_type=="E":
+                events["error"].append([file,line_number])
+                print_err(file,"[!] Checker %s got a error for %s on line %s"%(i,file,line_number),line=line_number)
+            else:
+                print_warn(i,"[!] Event decoding fail! Assuming checker %s is not wrapped calls compliant"%(i))
+                exec_status=2
+                break
+
+    if is_using_wrapped_syntax:
+        script_name=split_output[0]
+        description=split_output[1]
+        if len(events["error"])==0 and len(events["warn"])==0:
+            print("[+] Ran %s and got no warnings or errors"%(script_name))
+            exec_status=1
+        else:
+            print("[+] Ran %s, got %i errors and %i warnings"%(script_name,len(events["error"]),len(events["warn"])))
+            exec_status=0
+    else:
+        print("[+] Ran checker %s but finished with unknown status"%(i))
+        script_name=i
+
+    all_events.append([script_name,events,exec_status,description])
+
+all_pass=True
+failed_checks=[]
+unformatted_raw_output=[]
+table_content=""
+errors_encountered=""
+
+for name,events,exec_status,description in all_events:
+    
+    if exec_status==0:
+        table_content+=TABLE_FORMAT%(name,"No %s"%(CROSS_MARK_EMOJI))
+        all_pass=False
+        failed_checks.append([name,events,description])
+    elif exec_status==1:
+        table_content+=TABLE_FORMAT%(name,"Yes %s"%(CHECK_MARK_EMOJI))
+    elif exec_status==2:
+        table_content+=TABLE_FORMAT%(name,"Unknown %s"%(QUESTION_MARK_EMOJI))
+        unformatted_raw_output.append([name,events["raw_output"]])
+
+formatted_raw_output=[]
+for name,output in unformatted_raw_output:
+    formatted_raw_output.append(UNFORMATTED_OUTPUT_FORMAT%(name,output))
+
+formatted_raw_output='\n- - -\n'.join(formatted_raw_output)
+cleaned_failed_checks={}
+
+for name,events,description in failed_checks:
+
+    for err_type,err in events.items():
+        if err_type not in ["error","warn"]:
+            continue
+
+        for file,line_number in err:
+
+            if file not in cleaned_failed_checks.keys():
+                cleaned_failed_checks[file]={}
+                cleaned_failed_checks[file]["warn"]=[]
+                cleaned_failed_checks[file]["error"]=[]
+                cleaned_failed_checks[file]["check"]=name
+                cleaned_failed_checks[file]["description"]=description
+
+            cleaned_failed_checks[file][err_type].append(int(line_number))
+
+for file, warn_and_errors in cleaned_failed_checks.items():
+
+    warn=warn_and_errors["warn"]
+    error=warn_and_errors["error"]
+
+    for k,v in warn_and_errors.items():
+        
+        if k not in ["error","warn"]:
+            continue
+
+        v.sort()
+        lines=[]
+
+        for i in v:
+            i=int(i)
+
+            if not lines:
+                lines.append([i,i])
+                continue
+
+            if lines[-1][1]+1==i:
+                lines[-1][1]=i
+            else:
+                lines.append([i,i])
+
+        warn_and_errors[k]=lines
+
+if all_pass:
+    error_text="All good! No checks failed."
+
+else:
+
+    error_text=[]
+    check_results={}
+
+    for file,warn_and_errors in cleaned_failed_checks.items():
+        
+        error_msg=""
+        warn_msg=""
+        current_errors=[]
+        current_warnings=[]
+        checker_name=warn_and_errors["check"]
+        description=warn_and_errors["description"]
+
+        if checker_name not in check_results.keys():
+            check_results.update({checker_name:{"warn":[],"error":[]}})
+
+        for line_numbers in warn_and_errors["warn"]:
+            
+            line_numbers[0]=str(line_numbers[0])
+            line_numbers[1]=str(line_numbers[1])
+
+            if line_numbers[0]==line_numbers[1]:
+                current_warnings.append(line_numbers[0])
+                continue
+            
+            current_warnings.append('-'.join(line_numbers))
+
+        for line_numbers in warn_and_errors["error"]:
+
+            line_numbers[0]=str(line_numbers[0])
+            line_numbers[1]=str(line_numbers[1])
+
+            if line_numbers[0]==line_numbers[1]:
+                current_errors.append(line_numbers[0])
+                continue
+            
+            current_errors.append('-'.join(line_numbers))
+
+        if current_errors:
+            error_msg=ERROR_MSG%(file,', '.join(current_errors))
+            check_results[checker_name]["error"].append(error_msg)
+
+        if current_warnings:
+            warn_msg=WARN_MSG%(file,', '.join(current_warnings))
+            check_results[checker_name]["warn"].append(warn_msg)
+
+    for checker,results in check_results.items():
+    
+        if len(results["error"])>0:
+            error_msg='\n'.join(results["error"])
+        else:
+            error_msg="There are no errors for this check!"
+
+        if len(results["warn"])>0:
+            warn_msg='\n'.join(results["warn"])
+        else:
+            warn_msg="There are no warnings for this check!"
+        error_text.append(FORMATTED_OUTPUT_FORMAT%(checker,description,warn_msg,error_msg))
+
+    error_text='\n- - -\n'.join(error_text)
+
+open(STEP_SUMMARY_LOCATION,"w").write(SUMMARY_FORMAT%(table_content,error_text,formatted_raw_output))
+
+if not all_pass:
+    print_err(".bin/validators.py","[!] Not all checks passed.")
+    exit(2)

.bin/validators.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-# https://stackoverflow.com/questions/3822621/how-to-exit-if-a-command-failed
-
-set -e 
-set -o pipefail
-
-# wrapper for all the checking scripts
-echo $1
-./.bin/check-file-for-starting-slash "$1"
-./.bin/new-line-and-empty-line-checker.py "$1"

.github/workflows/wordlist-validator.yml

@@ -28,4 +28,4 @@ jobs:
         uses: tj-actions/changed-files@v34
       - name: Analyze all added or modified files
         run: |
-          ./.bin/validators.sh "${{ steps.changed-files.outputs.all_changed_files }}"
+          ./.bin/validators.py "${{ steps.changed-files.outputs.all_changed_files }}"