mirror of
https://github.com/danielmiessler/SecLists.git
synced 2024-09-11 10:55:33 +03:00
Reimport files
This commit is contained in:
parent
28c116e49d
commit
5fbcb2fba3
@ -1,17 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Script to verify if a file contain any entries starting with a slash and show a warning into the github action result UI.
|
||||
# Received as input a list of file paths to check separated by a space.
|
||||
# More precisely the result of the "tj-actions/changed-files" github action.
|
||||
## References:
|
||||
# See https://github.com/tj-actions/changed-files
|
||||
modified_files="$1"
|
||||
for modified_file in $modified_files
|
||||
do
|
||||
echo "[+] Check $modified_file ..."
|
||||
matches=$(grep -cE '^/[a-zA-Z0-9\._]+' $modified_file)
|
||||
echo "Entries identified starting with a slash: $matches"
|
||||
if [ $matches -ne 0 ]
|
||||
then
|
||||
echo "::warning file=$modified_file,line=1,col=1,endColumn=1::$matches entries start with a slash."
|
||||
fi
|
||||
done
|
45
.bin/checkers/README.md
Normal file
45
.bin/checkers/README.md
Normal file
@ -0,0 +1,45 @@
|
||||
All the files here will be run by `validators.sh` as part of the check pipeline
|
||||
|
||||
## Specifications
|
||||
Scripts will be passed a space separated list of files to check
|
||||
e.g `./script.sh "Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt"`
|
||||
|
||||
If you want your output to be parsed by the caller script, follow the below specs,else the output will be displayed raw in actions summaries
|
||||
|
||||
## Wrapped calls
|
||||
|
||||
### Wrapped calls checks
|
||||
|
||||
- - -
|
||||
|
||||
Scripts should check if its being run by the caller script. The environment variable `IS_RUNNING_UNDER_CALLER_SCRIPT` will be set to one
|
||||
|
||||
### Wrapped calls Specifications
|
||||
|
||||
- - -
|
||||
|
||||
Checker scripts will now have to print out the name of the check followed by a new line
|
||||
|
||||
Example `New line checker`
|
||||
|
||||
This value will be used as the checker title
|
||||
|
||||
After that, the descriptions will need to be printed out
|
||||
|
||||
Example `To fix the error, you have to remove the empty lines or new lines at the end of the file`
|
||||
|
||||
This is for contributors to understand why the checks failed
|
||||
|
||||
- - -
|
||||
|
||||
Outputs from now on will have to be in separate lines for each warnings or errors
|
||||
|
||||
This is the format for errors
|
||||
|
||||
Example `E,filename,line_number`
|
||||
|
||||
In the above example, `E` stands for errors. Accepted values are `E` and `W` for errors and warnings respectively
|
||||
|
||||
filename is the name of the file that the error comes from.
|
||||
|
||||
line_number is the line the script detected the error in.
|
66
.bin/checkers/check-file-for-starting-slash.py
Executable file
66
.bin/checkers/check-file-for-starting-slash.py
Executable file
@ -0,0 +1,66 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import os,sys
|
||||
|
||||
if not sys.argv[1]:
|
||||
exit(0)
|
||||
|
||||
IS_WRAPPED=False
|
||||
|
||||
if "IS_RUNNING_UNDER_CALLER_SCRIPT" in os.environ:
|
||||
IS_WRAPPED=os.environ['IS_RUNNING_UNDER_CALLER_SCRIPT']=="1"
|
||||
|
||||
def print_normal(msg):
|
||||
|
||||
if IS_WRAPPED:
|
||||
return
|
||||
print(msg)
|
||||
|
||||
def print_err(file,line_number):
|
||||
|
||||
if IS_WRAPPED:
|
||||
print("E,%s,%s"%(file,line_number))
|
||||
|
||||
def print_warn(file,line_number):
|
||||
|
||||
if IS_WRAPPED:
|
||||
print("W,%s,%s"%(file,line_number))
|
||||
|
||||
print_normal("[+] Entries starting with slash checker")
|
||||
if IS_WRAPPED:
|
||||
print("Entries starting with slash check")
|
||||
print("The warning are more for informative purposes and does not actually serve as a check. You can ignore this.")
|
||||
|
||||
files=sys.argv[1].split(" ")
|
||||
|
||||
for i in files:
|
||||
if not os.path.isfile(i):
|
||||
print_err(i,0)
|
||||
print_normal("[!] %s does not exist!"%(i))
|
||||
exit(2)
|
||||
|
||||
pass_status=True
|
||||
|
||||
for i in files:
|
||||
contents=open(i,"rb").read()
|
||||
counter=1
|
||||
|
||||
for line in contents.splitlines():
|
||||
if line.startswith(b'/'):
|
||||
print_normal("[!] Warning: %s starts with a slash on line %i!"%(i,counter))
|
||||
print_warn(i,counter)
|
||||
pass_status=False
|
||||
|
||||
counter+=1
|
||||
print_normal("[+] %s passed no starting slash check!"%(i))
|
||||
|
||||
if pass_status:
|
||||
print_normal("[+] All files passed checks")
|
||||
exit(0)
|
||||
|
||||
print_normal("[!] Warning: One or more files failed to pass the checks")
|
||||
|
||||
if IS_WRAPPED:
|
||||
exit(0)
|
||||
else:
|
||||
exit(2)
|
88
.bin/checkers/new-line-and-empty-line-checker.py
Executable file
88
.bin/checkers/new-line-and-empty-line-checker.py
Executable file
@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os,sys
|
||||
|
||||
if not sys.argv[1]:
|
||||
exit(0)
|
||||
|
||||
IS_WRAPPED=False
|
||||
|
||||
if "IS_RUNNING_UNDER_CALLER_SCRIPT" in os.environ:
|
||||
IS_WRAPPED=os.environ['IS_RUNNING_UNDER_CALLER_SCRIPT']=="1"
|
||||
|
||||
def print_normal(msg):
|
||||
|
||||
if IS_WRAPPED:
|
||||
return
|
||||
print(msg)
|
||||
|
||||
def print_err(file,line_number):
|
||||
|
||||
if IS_WRAPPED:
|
||||
print("E,%s,%s"%(file,line_number))
|
||||
|
||||
def print_warn(file,line_number):
|
||||
|
||||
if IS_WRAPPED:
|
||||
print("W,%s,%s"%(file,line_number))
|
||||
|
||||
print_normal("[+] New line and empty line check")
|
||||
if IS_WRAPPED:
|
||||
print("New line and empty line check")
|
||||
print("To fix the error, you would have to remove the empty lines or new lines at the end of the file.")
|
||||
|
||||
files=sys.argv[1].split(" ")
|
||||
|
||||
for i in files:
|
||||
if not os.path.isfile(i):
|
||||
print_err(i,0)
|
||||
print_normal("[!] %s does not exist!"%(i))
|
||||
exit(2)
|
||||
|
||||
overall_pass_status=True
|
||||
|
||||
for i in files:
|
||||
|
||||
contents=open(i,"rb").read()
|
||||
line_number=len(contents.split(b'\n'))+1
|
||||
|
||||
if contents[-1:] == b'\n':
|
||||
print_normal("[!] Warning: %s ends with a new line!"%(i))
|
||||
print_warn(i,line_number)
|
||||
overall_pass_status=False
|
||||
else:
|
||||
print_normal("[+] %s passed new line check!"%(i))
|
||||
|
||||
counter=1
|
||||
|
||||
line_pass_status=True
|
||||
|
||||
for line in contents.splitlines():
|
||||
if not line:
|
||||
print_normal("[!] Warning: %s has an empty entry on line %i!"%(i,counter))
|
||||
print_warn(i,counter)
|
||||
pass_status=False
|
||||
line_pass_status=False
|
||||
continue
|
||||
|
||||
elif not line.strip():
|
||||
print_normal("[!] Warning: %s has an whitespace only entry on line %i!"%(i,counter))
|
||||
print_warn(i,counter)
|
||||
pass_status=False
|
||||
line_pass_status=False
|
||||
continue
|
||||
|
||||
counter+=1
|
||||
if line_pass_status:
|
||||
print_normal("[+] %s passed empty line check!"%(i))
|
||||
|
||||
if overall_pass_status:
|
||||
print_normal("[+] All files passed checks")
|
||||
exit(0)
|
||||
|
||||
print_normal("[!] Warning: One or more files failed to pass the checks")
|
||||
|
||||
if IS_WRAPPED:
|
||||
exit(0)
|
||||
else:
|
||||
exit(2)
|
@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# test string: ./.bin/new-line-checker.py "Fuzzing/file-extensions-all-cases.txt Fuzzing/file-extensions-lower-case.txt Fuzzing/file-extensions-upper-case.txt Fuzzing/file-extensions.txt"
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
print("[+] New line check")
|
||||
|
||||
if not sys.argv[1]:
|
||||
exit(0)
|
||||
|
||||
files=sys.argv[1].split(" ")
|
||||
|
||||
for i in files:
|
||||
if not os.path.isfile(i):
|
||||
print("[!] %s does not exist!"%(i))
|
||||
exit(2)
|
||||
|
||||
for i in files:
|
||||
contents=open(i,"rb").read()
|
||||
|
||||
if contents[-1] == b'\n':
|
||||
print("[!] %s ends with a new line!"%(i))
|
||||
exit(2)
|
||||
print("[+] %s passed new line check!"%(i))
|
||||
|
||||
counter=1
|
||||
|
||||
for line in contents.splitlines(False):
|
||||
if len(line)==0:
|
||||
print("[!] %s has an empty entry at line %i!"%(i,counter))
|
||||
exit(2)
|
||||
counter+=1
|
||||
print("[+] %s passed empty line check!"%(i))
|
||||
|
||||
print("[+] All files passed checks")
|
||||
# exit(0)
|
323
.bin/validators.py
Executable file
323
.bin/validators.py
Executable file
@ -0,0 +1,323 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
# The workflow commands are from https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions
|
||||
# TL;DR You can echo text out and the text can trigger workflow commands if in format "::workflow-command parameter1={data},parameter2={data}::{command value}"
|
||||
# We are using the warnings and errors settings.
|
||||
# You may use the below lines for testing
|
||||
#
|
||||
# Invalid test args ./.bin/validators.py "Fuzzing/file-extensions-all-cases.txt Fuzzing/file-extensions-lower-case.txt Fuzzing/file-extensions-upper-case.txt Fuzzing/file-extensions.txt"
|
||||
# valid test args: ./.bin/validators.py "Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt"
|
||||
#
|
||||
# Script by @molangning. I am the guy you want to ping when this script get borked, not the repository owners
|
||||
|
||||
import os,subprocess,sys
|
||||
|
||||
args=sys.argv[1]
|
||||
files=[]
|
||||
STEP_SUMMARY_LOCATION="summary.md"
|
||||
IS_RUNNING_AS_ACTIONS=False
|
||||
CHECK_MARK_EMOJI=":white_check_mark:"
|
||||
CROSS_MARK_EMOJI=":negative_squared_cross_mark:"
|
||||
QUESTION_MARK_EMOJI=":question:"
|
||||
FORMATTED_OUTPUT_FORMAT="""
|
||||
### %s
|
||||
|
||||
%s
|
||||
|
||||
- - -
|
||||
|
||||
Errors
|
||||
|
||||
```
|
||||
%s
|
||||
```
|
||||
|
||||
Warnings
|
||||
|
||||
```
|
||||
%s
|
||||
```
|
||||
"""
|
||||
UNFORMATTED_OUTPUT_FORMAT="""
|
||||
### %s
|
||||
|
||||
- - -
|
||||
|
||||
Output
|
||||
|
||||
```
|
||||
%s
|
||||
```
|
||||
"""
|
||||
SUMMARY_FORMAT="""
|
||||
# Checks summary
|
||||
|
||||
This is a summary of the checks ran on the pushed files. If there is any errors please fix them before pushing again.
|
||||
## Check statuses
|
||||
|
||||
| Test name | Passed? |
|
||||
| --------- | :-----: |
|
||||
%s
|
||||
|
||||
## Errors and warnings
|
||||
%s
|
||||
|
||||
## Other unformatted outputs
|
||||
This section is for scripts that doesn't follow specifications
|
||||
|
||||
%s
|
||||
"""
|
||||
TABLE_FORMAT="| %s | %s |\n"
|
||||
WARN_MSG="Warnings in file %s on lines %s"
|
||||
ERROR_MSG="Errors in file %s on lines %s"
|
||||
WARNING_STRING="::warning file=%s,line=%s,col=%s,endColumn=%s::%s"
|
||||
ERROR_STRING="::error file=%s,line=%s,col=%s,endColumn=%s::%s"
|
||||
|
||||
if "GITHUB_STEP_SUMMARY" not in os.environ:
|
||||
print("[!] GITHUB_STEP_SUMMARY not found in system environments!")
|
||||
print("[-] This error may occur if you are running this script in your own machine\n")
|
||||
else:
|
||||
STEP_SUMMARY_LOCATION=os.environ["GITHUB_STEP_SUMMARY"]
|
||||
IS_RUNNING_AS_ACTIONS=True
|
||||
|
||||
if IS_RUNNING_AS_ACTIONS:
|
||||
print("[+] Now running as github actions\n")
|
||||
else:
|
||||
print("[+] Now running as normal user\n")
|
||||
|
||||
def print_warn(file,msg,line=1,col=1,endcol=1):
|
||||
if IS_RUNNING_AS_ACTIONS:
|
||||
print(WARNING_STRING%(file,line,col,endcol,msg))
|
||||
else:
|
||||
print(msg)
|
||||
|
||||
|
||||
def print_err(file,msg,line=1,col=1,endcol=1):
|
||||
if IS_RUNNING_AS_ACTIONS:
|
||||
print(ERROR_STRING%(file,line,col,endcol,msg))
|
||||
else:
|
||||
print(msg)
|
||||
|
||||
for root,_,file_list in os.walk('.bin/checkers'):
|
||||
for file in file_list:
|
||||
if file.endswith('.md'):
|
||||
continue
|
||||
|
||||
files.append(os.path.join(root,file))
|
||||
|
||||
sys_env=os.environ.copy()
|
||||
sys_env.update({"IS_RUNNING_UNDER_CALLER_SCRIPT":"1"})
|
||||
all_events=[]
|
||||
|
||||
for i in files:
|
||||
events={"error":[],"warn":[],"raw_output":""}
|
||||
is_using_wrapped_syntax=True
|
||||
exec_status=1 # 0 for fail, 1 for success, 2 for unknown
|
||||
description=""
|
||||
|
||||
try:
|
||||
output=subprocess.check_output([i,args],env=sys_env)
|
||||
|
||||
except PermissionError:
|
||||
print_warn(i,"[!] Not running test %s due to insufficient privileges!"%(i))
|
||||
print_warn(i,"[!] chmod +x it to let the workflow run the check.\n")
|
||||
continue
|
||||
|
||||
except subprocess.CalledProcessError as exec_err:
|
||||
print_err(i,"[!] Error! Calling command with args %s failed!"%(str([i,args])))
|
||||
print_err(i,"[!] Process exited with error code %s and error message %s\n"%(exec_err.returncode,exec_err.output))
|
||||
continue
|
||||
|
||||
output=output.decode('utf-8')
|
||||
events["raw_output"]=output
|
||||
|
||||
split_output=output.split('\n')
|
||||
|
||||
if split_output[-1]=="":
|
||||
split_output=split_output[:-1]
|
||||
|
||||
if len(split_output)<2:
|
||||
print_warn(i,"[!] Checker printed out less than two lines! Assuming not wrapped calls compliant.")
|
||||
exec_status=2
|
||||
is_using_wrapped_syntax=False
|
||||
|
||||
else:
|
||||
|
||||
for line in split_output[2:]:
|
||||
|
||||
if not len(line):
|
||||
continue
|
||||
|
||||
try:
|
||||
event_type,file,line_number=line.split(',')
|
||||
except:
|
||||
print_warn(i,"[!] Split fail! Assuming checker %s is not wrapped calls compliant."%(i))
|
||||
is_using_wrapped_syntax=False
|
||||
exec_status=2
|
||||
break
|
||||
|
||||
if event_type=="W":
|
||||
events["warn"].append([file,line_number])
|
||||
print_err(file,"[!] Checker %s got a warning for %s on line %s"%(i,file,line_number),line=line_number)
|
||||
elif event_type=="E":
|
||||
events["error"].append([file,line_number])
|
||||
print_err(file,"[!] Checker %s got a error for %s on line %s"%(i,file,line_number),line=line_number)
|
||||
else:
|
||||
print_warn(i,"[!] Event decoding fail! Assuming checker %s is not wrapped calls compliant"%(i))
|
||||
exec_status=2
|
||||
break
|
||||
|
||||
if is_using_wrapped_syntax:
|
||||
script_name=split_output[0]
|
||||
description=split_output[1]
|
||||
if len(events["error"])==0 and len(events["warn"])==0:
|
||||
print("[+] Ran %s and got no warnings or errors"%(script_name))
|
||||
exec_status=1
|
||||
else:
|
||||
print("[+] Ran %s, got %i errors and %i warnings"%(script_name,len(events["error"]),len(events["warn"])))
|
||||
exec_status=0
|
||||
else:
|
||||
print("[+] Ran checker %s but finished with unknown status"%(i))
|
||||
script_name=i
|
||||
|
||||
all_events.append([script_name,events,exec_status,description])
|
||||
|
||||
all_pass=True
|
||||
failed_checks=[]
|
||||
unformatted_raw_output=[]
|
||||
table_content=""
|
||||
errors_encountered=""
|
||||
|
||||
for name,events,exec_status,description in all_events:
|
||||
|
||||
if exec_status==0:
|
||||
table_content+=TABLE_FORMAT%(name,"No %s"%(CROSS_MARK_EMOJI))
|
||||
all_pass=False
|
||||
failed_checks.append([name,events,description])
|
||||
elif exec_status==1:
|
||||
table_content+=TABLE_FORMAT%(name,"Yes %s"%(CHECK_MARK_EMOJI))
|
||||
elif exec_status==2:
|
||||
table_content+=TABLE_FORMAT%(name,"Unknown %s"%(QUESTION_MARK_EMOJI))
|
||||
unformatted_raw_output.append([name,events["raw_output"]])
|
||||
|
||||
formatted_raw_output=[]
|
||||
for name,output in unformatted_raw_output:
|
||||
formatted_raw_output.append(UNFORMATTED_OUTPUT_FORMAT%(name,output))
|
||||
|
||||
formatted_raw_output='\n- - -\n'.join(formatted_raw_output)
|
||||
cleaned_failed_checks={}
|
||||
|
||||
for name,events,description in failed_checks:
|
||||
|
||||
for err_type,err in events.items():
|
||||
if err_type not in ["error","warn"]:
|
||||
continue
|
||||
|
||||
for file,line_number in err:
|
||||
|
||||
if file not in cleaned_failed_checks.keys():
|
||||
cleaned_failed_checks[file]={}
|
||||
cleaned_failed_checks[file]["warn"]=[]
|
||||
cleaned_failed_checks[file]["error"]=[]
|
||||
cleaned_failed_checks[file]["check"]=name
|
||||
cleaned_failed_checks[file]["description"]=description
|
||||
|
||||
cleaned_failed_checks[file][err_type].append(int(line_number))
|
||||
|
||||
for file, warn_and_errors in cleaned_failed_checks.items():
|
||||
|
||||
warn=warn_and_errors["warn"]
|
||||
error=warn_and_errors["error"]
|
||||
|
||||
for k,v in warn_and_errors.items():
|
||||
|
||||
if k not in ["error","warn"]:
|
||||
continue
|
||||
|
||||
v.sort()
|
||||
lines=[]
|
||||
|
||||
for i in v:
|
||||
i=int(i)
|
||||
|
||||
if not lines:
|
||||
lines.append([i,i])
|
||||
continue
|
||||
|
||||
if lines[-1][1]+1==i:
|
||||
lines[-1][1]=i
|
||||
else:
|
||||
lines.append([i,i])
|
||||
|
||||
warn_and_errors[k]=lines
|
||||
|
||||
if all_pass:
|
||||
error_text="All good! No checks failed."
|
||||
|
||||
else:
|
||||
|
||||
error_text=[]
|
||||
check_results={}
|
||||
|
||||
for file,warn_and_errors in cleaned_failed_checks.items():
|
||||
|
||||
error_msg=""
|
||||
warn_msg=""
|
||||
current_errors=[]
|
||||
current_warnings=[]
|
||||
checker_name=warn_and_errors["check"]
|
||||
description=warn_and_errors["description"]
|
||||
|
||||
if checker_name not in check_results.keys():
|
||||
check_results.update({checker_name:{"warn":[],"error":[]}})
|
||||
|
||||
for line_numbers in warn_and_errors["warn"]:
|
||||
|
||||
line_numbers[0]=str(line_numbers[0])
|
||||
line_numbers[1]=str(line_numbers[1])
|
||||
|
||||
if line_numbers[0]==line_numbers[1]:
|
||||
current_warnings.append(line_numbers[0])
|
||||
continue
|
||||
|
||||
current_warnings.append('-'.join(line_numbers))
|
||||
|
||||
for line_numbers in warn_and_errors["error"]:
|
||||
|
||||
line_numbers[0]=str(line_numbers[0])
|
||||
line_numbers[1]=str(line_numbers[1])
|
||||
|
||||
if line_numbers[0]==line_numbers[1]:
|
||||
current_errors.append(line_numbers[0])
|
||||
continue
|
||||
|
||||
current_errors.append('-'.join(line_numbers))
|
||||
|
||||
if current_errors:
|
||||
error_msg=ERROR_MSG%(file,', '.join(current_errors))
|
||||
check_results[checker_name]["error"].append(error_msg)
|
||||
|
||||
if current_warnings:
|
||||
warn_msg=WARN_MSG%(file,', '.join(current_warnings))
|
||||
check_results[checker_name]["warn"].append(warn_msg)
|
||||
|
||||
for checker,results in check_results.items():
|
||||
|
||||
if len(results["error"])>0:
|
||||
error_msg='\n'.join(results["error"])
|
||||
else:
|
||||
error_msg="There are no errors for this check!"
|
||||
|
||||
if len(results["warn"])>0:
|
||||
warn_msg='\n'.join(results["warn"])
|
||||
else:
|
||||
warn_msg="There are no warnings for this check!"
|
||||
error_text.append(FORMATTED_OUTPUT_FORMAT%(checker,description,warn_msg,error_msg))
|
||||
|
||||
error_text='\n- - -\n'.join(error_text)
|
||||
|
||||
open(STEP_SUMMARY_LOCATION,"w").write(SUMMARY_FORMAT%(table_content,error_text,formatted_raw_output))
|
||||
|
||||
if not all_pass:
|
||||
print_err(".bin/validators.py","[!] Not all checks passed.")
|
||||
exit(2)
|
@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# https://stackoverflow.com/questions/3822621/how-to-exit-if-a-command-failed
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
# wrapper for all the checking scripts
|
||||
echo $1
|
||||
./.bin/check-file-for-starting-slash "$1"
|
||||
./.bin/new-line-and-empty-line-checker.py "$1"
|
2
.github/workflows/wordlist-validator.yml
vendored
2
.github/workflows/wordlist-validator.yml
vendored
@ -28,4 +28,4 @@ jobs:
|
||||
uses: tj-actions/changed-files@v34
|
||||
- name: Analyze all added or modified files
|
||||
run: |
|
||||
./.bin/validators.sh "${{ steps.changed-files.outputs.all_changed_files }}"
|
||||
./.bin/validators.py "${{ steps.changed-files.outputs.all_changed_files }}"
|
||||
|
Loading…
Reference in New Issue
Block a user