Merge pull request #785 from nicholas-long/master

Create list of files in that could go in /etc for fuzzing

.bin/etc-files-list-update/README.md

@@ -0,0 +1,17 @@
+# Etc File List Updater
+
+## Overview
+The purpose of this set of scripts is to update the file `Fuzzing/LFI/LFI-etc-files-of-all-linux-packages.txt`.
+It is intended to be run periodically.
+The scripts scan all deb packages in the ubuntu apt repository that have been updated since the last scan.
+URLs that have been scanned will be added to a data file that can be stored in git.
+
+## Running
+The script must be run from its working directory.
+```bash
+cd .bin/etc-files-list-update && ./update.sh
+```
+
+## Details
+URLs for deb files that have already been scanned are stored in gzip format in the `deb-url-history/` directory.
+The current ubuntu distro for which packages are retrieved is stored in the file `current_distro`. This should be changed every few years.

.bin/etc-files-list-update/current_distro

@@ -0,0 +1 @@
+kinetic

.bin/etc-files-list-update/update.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+export listpath="../../Fuzzing/LFI/LFI-etc-files-of-all-linux-packages.txt"
+[ -f all_files.gz ] && rm all_files.gz
+
+# every year, start a new gzip list so there is not as much bloat updating blobs in git
+year=$(date +%Y)
+
+echo "finding URLs" 1>&2
+
+# get new URLs
+util/find-new-urls.awk > url_batch
+
+# exit if there's no new URLs to scan
+if [[ $(wc -l url_batch | awk '{print $1}') == 0 ]]
+then
+  echo "no new URLs" 1>&2
+  rm url_batch
+  exit 0
+fi
+
+# scan them
+for u in $(cat url_batch)
+do
+  echo "scanning $u" 1>&2
+  util/scan-package.sh "$u" | gzip >> all_files.gz
+done
+
+echo "searching for etc files" 1>&2
+
+# get all files matching /etc/
+# ignore repeat files already in the list
+zcat all_files.gz | awk '
+BEGIN {
+  lp = ENVIRON["listpath"]
+  while (getline < lp) {
+    seen[$0] = 1
+  }
+}
+/^\/etc\// && !seen[$0] { print }
+' > updated_etc_files
+
+echo "updating list" 1>&2
+
+# concatenate the existing list and the output
+cat "$listpath" updated_etc_files > updated_file
+
+# update the list
+mv updated_file "$listpath"
+
+# save progress
+cat url_batch | gzip >> "deb-url-history/$year.gz"
+
+# cleanup
+rm url_batch
+rm updated_etc_files
+rm all_files.gz

.bin/etc-files-list-update/util/find-new-urls.awk

@@ -0,0 +1,14 @@
+#!/usr/bin/awk -f
+BEGIN {
+  # load all the URLs we scanned already
+  command = "util/print-urls.sh"
+  while (command | getline) {
+    urls[$0] = 1 # add to set
+  }
+  close(command)
+  # get package URLs that do not appear in the list
+  command = "util/get-package-urls.sh"
+  while (command | getline) {
+    if (!($0 in urls)) print
+  }
+}

.bin/etc-files-list-update/util/get-package-urls.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# get new package URLs
+# load the list of amd64 packages from ubuntu
+export dist="$(cat current_distro)"
+export repo="http://archive.ubuntu.com/ubuntu"
+
+# print URLs
+curl $repo/dists/$dist/main/binary-amd64/Packages.gz | \
+  gzip -d | awk '/^Filename: / { print ENVIRON["repo"] "/" $2 }'

.bin/etc-files-list-update/util/print-urls.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# print every url in every file in deb-url-history directory
+for f in $(ls deb-url-history/)
+do
+  zcat "deb-url-history/$f"
+done

.bin/etc-files-list-update/util/scan-package.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+export url=$1
+
+tf=$(mktemp -d)
+wd=$(pwd)
+cd $tf
+wget "$url" -O output 2>/dev/null >/dev/null
+ar -x output # extracts data.tar.xz control.tar.xz
+
+# extract tar
+if [ -f control.tar.xz ]; then
+  xz -d control.tar.xz 2>/dev/null
+elif [ -f control.tar.zst ]; then # need to install zstd
+  zstd -d control.tar.zst 2>/dev/null
+elif [ -f control.tar.gz ]; then
+  tar -xzvf control.tar.gz 2>/dev/null >/dev/null
+else
+  (echo "$url unknown deb compression format" && ls) >> problems
+  exit 0
+fi
+
+# extract control
+tar -xvf control.tar 2>/dev/null >/dev/null
+
+# replace 2 spaces after md5sum with tab
+sed 's/^\([0-9a-zA-Z]*\)  /\1\t/' md5sums > inputdata
+
+# print filenames
+awk '
+BEGIN {
+  FS="\t"
+}
+{
+  gsub(/^\.\//,"",$2)
+  print "/" $2
+}
+' inputdata
+
+# cleanup
+cd "$wd"
+rm -rf $tf

.github/workflows/wordlist-updater_fuzzing_etc_files.yml

@@ -0,0 +1,29 @@
+name: update etc files
+
+# Controls when the workflow will run
+on:
+  schedule:
+    - cron: '30 20 1,15 * *' # run at 8:30p on 1st and 15th
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  updatejob:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      - uses: actions/checkout@v3
+
+      # Runs a single command using the runners shell
+      - name: update wordlist
+        run: cd .bin/etc-files-list-update/ && ./update.sh
+
+      - name: print diff
+        run: git diff
+
+      # commit and push
+      - uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: '[Github Action] Updated LFI-etc-files-of-all-linux-packages.txt'