mirror of
https://github.com/danielmiessler/SecLists.git
synced 2024-09-11 10:55:33 +03:00
Merge pull request #785 from nicholas-long/master
Create list of files in that could go in /etc for fuzzing
This commit is contained in:
commit
b49003fdbc
17
.bin/etc-files-list-update/README.md
Normal file
17
.bin/etc-files-list-update/README.md
Normal file
@ -0,0 +1,17 @@
|
||||
# Etc File List Updater
|
||||
|
||||
## Overview
|
||||
The purpose of this set of scripts is to update the file `Fuzzing/LFI/LFI-etc-files-of-all-linux-packages.txt`.
|
||||
It is intended to be run periodically.
|
||||
The scripts scan all deb packages in the ubuntu apt repository that have been updated since the last scan.
|
||||
URLs that have been scanned will be added to a data file that can be stored in git.
|
||||
|
||||
## Running
|
||||
The script must be run from its working directory.
|
||||
```bash
|
||||
cd .bin/etc-files-list-update && ./update.sh
|
||||
```
|
||||
|
||||
## Details
|
||||
URLs for deb files that have already been scanned are stored in gzip format in the `deb-url-history/` directory.
|
||||
The current ubuntu distro for which packages are retrieved is stored in the file `current_distro`. This should be changed every few years.
|
1
.bin/etc-files-list-update/current_distro
Normal file
1
.bin/etc-files-list-update/current_distro
Normal file
@ -0,0 +1 @@
|
||||
kinetic
|
BIN
.bin/etc-files-list-update/deb-url-history/2022.gz
Normal file
BIN
.bin/etc-files-list-update/deb-url-history/2022.gz
Normal file
Binary file not shown.
BIN
.bin/etc-files-list-update/deb-url-history/deb.urls.initial.gz
Normal file
BIN
.bin/etc-files-list-update/deb-url-history/deb.urls.initial.gz
Normal file
Binary file not shown.
57
.bin/etc-files-list-update/update.sh
Executable file
57
.bin/etc-files-list-update/update.sh
Executable file
@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
|
||||
export listpath="../../Fuzzing/LFI/LFI-etc-files-of-all-linux-packages.txt"
|
||||
[ -f all_files.gz ] && rm all_files.gz
|
||||
|
||||
# every year, start a new gzip list so there is not as much bloat updating blobs in git
|
||||
year=$(date +%Y)
|
||||
|
||||
echo "finding URLs" 1>&2
|
||||
|
||||
# get new URLs
|
||||
util/find-new-urls.awk > url_batch
|
||||
|
||||
# exit if there's no new URLs to scan
|
||||
if [[ $(wc -l url_batch | awk '{print $1}') == 0 ]]
|
||||
then
|
||||
echo "no new URLs" 1>&2
|
||||
rm url_batch
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# scan them
|
||||
for u in $(cat url_batch)
|
||||
do
|
||||
echo "scanning $u" 1>&2
|
||||
util/scan-package.sh "$u" | gzip >> all_files.gz
|
||||
done
|
||||
|
||||
echo "searching for etc files" 1>&2
|
||||
|
||||
# get all files matching /etc/
|
||||
# ignore repeat files already in the list
|
||||
zcat all_files.gz | awk '
|
||||
BEGIN {
|
||||
lp = ENVIRON["listpath"]
|
||||
while (getline < lp) {
|
||||
seen[$0] = 1
|
||||
}
|
||||
}
|
||||
/^\/etc\// && !seen[$0] { print }
|
||||
' > updated_etc_files
|
||||
|
||||
echo "updating list" 1>&2
|
||||
|
||||
# concatenate the existing list and the output
|
||||
cat "$listpath" updated_etc_files > updated_file
|
||||
|
||||
# update the list
|
||||
mv updated_file "$listpath"
|
||||
|
||||
# save progress
|
||||
cat url_batch | gzip >> "deb-url-history/$year.gz"
|
||||
|
||||
# cleanup
|
||||
rm url_batch
|
||||
rm updated_etc_files
|
||||
rm all_files.gz
|
14
.bin/etc-files-list-update/util/find-new-urls.awk
Executable file
14
.bin/etc-files-list-update/util/find-new-urls.awk
Executable file
@ -0,0 +1,14 @@
|
||||
#!/usr/bin/awk -f
|
||||
BEGIN {
|
||||
# load all the URLs we scanned already
|
||||
command = "util/print-urls.sh"
|
||||
while (command | getline) {
|
||||
urls[$0] = 1 # add to set
|
||||
}
|
||||
close(command)
|
||||
# get package URLs that do not appear in the list
|
||||
command = "util/get-package-urls.sh"
|
||||
while (command | getline) {
|
||||
if (!($0 in urls)) print
|
||||
}
|
||||
}
|
10
.bin/etc-files-list-update/util/get-package-urls.sh
Executable file
10
.bin/etc-files-list-update/util/get-package-urls.sh
Executable file
@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
# get new package URLs
|
||||
# load the list of amd64 packages from ubuntu
|
||||
export dist="$(cat current_distro)"
|
||||
export repo="http://archive.ubuntu.com/ubuntu"
|
||||
|
||||
# print URLs
|
||||
curl $repo/dists/$dist/main/binary-amd64/Packages.gz | \
|
||||
gzip -d | awk '/^Filename: / { print ENVIRON["repo"] "/" $2 }'
|
7
.bin/etc-files-list-update/util/print-urls.sh
Executable file
7
.bin/etc-files-list-update/util/print-urls.sh
Executable file
@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
# print every url in every file in deb-url-history directory
|
||||
for f in $(ls deb-url-history/)
|
||||
do
|
||||
zcat "deb-url-history/$f"
|
||||
done
|
42
.bin/etc-files-list-update/util/scan-package.sh
Executable file
42
.bin/etc-files-list-update/util/scan-package.sh
Executable file
@ -0,0 +1,42 @@
|
||||
#!/bin/bash
|
||||
|
||||
export url=$1
|
||||
|
||||
tf=$(mktemp -d)
|
||||
wd=$(pwd)
|
||||
cd $tf
|
||||
wget "$url" -O output 2>/dev/null >/dev/null
|
||||
ar -x output # extracts data.tar.xz control.tar.xz
|
||||
|
||||
# extract tar
|
||||
if [ -f control.tar.xz ]; then
|
||||
xz -d control.tar.xz 2>/dev/null
|
||||
elif [ -f control.tar.zst ]; then # need to install zstd
|
||||
zstd -d control.tar.zst 2>/dev/null
|
||||
elif [ -f control.tar.gz ]; then
|
||||
tar -xzvf control.tar.gz 2>/dev/null >/dev/null
|
||||
else
|
||||
(echo "$url unknown deb compression format" && ls) >> problems
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# extract control
|
||||
tar -xvf control.tar 2>/dev/null >/dev/null
|
||||
|
||||
# replace 2 spaces after md5sum with tab
|
||||
sed 's/^\([0-9a-zA-Z]*\) /\1\t/' md5sums > inputdata
|
||||
|
||||
# print filenames
|
||||
awk '
|
||||
BEGIN {
|
||||
FS="\t"
|
||||
}
|
||||
{
|
||||
gsub(/^\.\//,"",$2)
|
||||
print "/" $2
|
||||
}
|
||||
' inputdata
|
||||
|
||||
# cleanup
|
||||
cd "$wd"
|
||||
rm -rf $tf
|
29
.github/workflows/wordlist-updater_fuzzing_etc_files.yml
vendored
Normal file
29
.github/workflows/wordlist-updater_fuzzing_etc_files.yml
vendored
Normal file
@ -0,0 +1,29 @@
|
||||
name: update etc files
|
||||
|
||||
# Controls when the workflow will run
|
||||
on:
|
||||
schedule:
|
||||
- cron: '30 20 1,15 * *' # run at 8:30p on 1st and 15th
|
||||
|
||||
# Allows you to run this workflow manually from the Actions tab
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
updatejob:
|
||||
# The type of runner that the job will run on
|
||||
runs-on: ubuntu-latest
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
# Runs a single command using the runners shell
|
||||
- name: update wordlist
|
||||
run: cd .bin/etc-files-list-update/ && ./update.sh
|
||||
|
||||
- name: print diff
|
||||
run: git diff
|
||||
|
||||
# commit and push
|
||||
- uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
commit_message: '[Github Action] Updated LFI-etc-files-of-all-linux-packages.txt'
|
8314
Fuzzing/LFI/LFI-etc-files-of-all-linux-packages.txt
Normal file
8314
Fuzzing/LFI/LFI-etc-files-of-all-linux-packages.txt
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user